From 18b523912d27d8b8a863b3cbcf77563b02790581 Mon Sep 17 00:00:00 2001 From: KM Koushik Date: Mon, 10 Jun 2024 17:40:42 +1000 Subject: [PATCH] Improve self host support (#28) * Add docker setup for self hosting * Add ses settings tables --- Dockerfile | 76 ++++++ apps/web/.env.example | 25 -- apps/web/next.config.js | 4 +- apps/web/package.json | 3 +- .../migration.sql | 25 ++ apps/web/prisma/schema.prisma | 20 ++ .../src/app/(dashboard)/api-keys/api-list.tsx | 6 + apps/web/src/app/(dashboard)/layout.tsx | 8 +- apps/web/src/app/(dashboard)/nav-button.tsx | 14 ++ apps/web/src/app/api/ses_callback/route.ts | 34 +++ apps/web/src/app/layout.tsx | 9 +- apps/web/src/app/login/login-page.tsx | 216 ++++++++++-------- apps/web/src/env.js | 15 +- apps/web/src/server/api/routers/admin.ts | 37 +++ apps/web/src/server/api/trpc.ts | 11 + apps/web/src/server/auth.ts | 80 ++++--- apps/web/src/server/mailer.ts | 15 +- apps/web/src/server/public-api/auth.ts | 2 + apps/web/src/server/service/job-service.ts | 2 +- .../server/service/ses-settings-service.ts | 195 ++++++++++++++++ docker-compose.yml | 56 +++++ pnpm-lock.yaml | 9 + start.sh | 12 + turbo.json | 3 +- 24 files changed, 708 insertions(+), 169 deletions(-) create mode 100644 Dockerfile delete mode 100644 apps/web/.env.example create mode 100644 apps/web/prisma/migrations/20240610073426_add_ses_settings/migration.sql create mode 100644 apps/web/src/server/api/routers/admin.ts create mode 100644 apps/web/src/server/service/ses-settings-service.ts create mode 100644 docker-compose.yml create mode 100644 start.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..117dac3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,76 @@ +FROM node:20.11.1-alpine AS base +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +ENV SKIP_ENV_VALIDATION="true" +ENV DOCKER_OUTPUT 1 +ENV NEXT_TELEMETRY_DISABLED 1 + +RUN corepack enable + +FROM base AS builder +RUN apk add --no-cache libc6-compat +RUN apk update +# Set working directory +WORKDIR /app +# Replace with the major version installed in your repository. For example: +# RUN yarn global add turbo@^2 +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json start.sh ./ +COPY ./apps/web ./apps/web +COPY ./packages ./packages +RUN pnpm add turbo@^1.12.5 -g + +# Generate a partial monorepo with a pruned lockfile for a target workspace. +# Assuming "web" is the name entered in the project's package.json: { name: "web" } +RUN pnpm turbo prune web --docker + +# Add lockfile and package.json's of isolated subworkspace +FROM base AS installer +RUN apk add --no-cache libc6-compat +RUN apk update +WORKDIR /app + + +# First install the dependencies (as they change less often) +COPY .gitignore .gitignore +COPY --from=builder /app/out/json/ . +COPY --from=builder /app/out/pnpm-lock.yaml ./pnpm-lock.yaml +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile + + +# Build the project +COPY --from=builder /app/out/full/ . + +RUN pnpm turbo run build --filter=web... + +FROM base AS runner +WORKDIR /app + + + +COPY --from=installer /app/apps/web/next.config.js . +COPY --from=installer /app/apps/web/package.json . +COPY --from=installer /app/pnpm-lock.yaml . + + +# Automatically leverage output traces to reduce image size +# https://nextjs.org/docs/advanced-features/output-file-tracing +COPY --from=installer /app/apps/web/.next/standalone ./ +COPY --from=installer /app/apps/web/.next/static ./apps/web/.next/static +COPY --from=installer /app/apps/web/public ./apps/web/public + +# Copy prisma files +COPY --from=installer /app/apps/web/prisma/schema.prisma ./apps/web/prisma/schema.prisma +COPY --from=installer /app/apps/web/prisma/migrations ./apps/web/prisma/migrations +COPY --from=installer /app/apps/web/node_modules/prisma ./node_modules/prisma +COPY --from=installer /app/apps/web/node_modules/@prisma ./node_modules/@prisma + +# Symlink the prisma binary +RUN mkdir node_modules/.bin +RUN ln -s /app/node_modules/prisma/build/index.js ./node_modules/.bin/prisma + +# set this so it throws error where starting server +ENV SKIP_ENV_VALIDATION="false" + +COPY start.sh ./ + +CMD ["sh", "start.sh"] \ No newline at end of file diff --git a/apps/web/.env.example b/apps/web/.env.example deleted file mode 100644 index 620cd4e..0000000 --- a/apps/web/.env.example +++ /dev/null @@ -1,25 +0,0 @@ -# Since the ".env" file is gitignored, you can use the ".env.example" file to -# build a new ".env" file when you clone the repo. Keep this file up-to-date -# when you add new variables to `.env`. - -# This file will be committed to version control, so make sure not to have any -# secrets in it. If you are cloning this repo, create a copy of this file named -# ".env" and populate it with your secrets. - -# When adding additional environment variables, the schema in "/src/env.js" -# should be updated accordingly. - -# Prisma -# https://www.prisma.io/docs/reference/database-reference/connection-urls#env -DATABASE_URL="postgresql://postgres:password@localhost:5432/web" - -# Next Auth -# You can generate a new secret on the command line with: -# openssl rand -base64 32 -# https://next-auth.js.org/configuration/options#secret -# NEXTAUTH_SECRET="" -NEXTAUTH_URL="http://localhost:3000" - -# Next Auth Discord Provider -DISCORD_CLIENT_ID="" -DISCORD_CLIENT_SECRET="" diff --git a/apps/web/next.config.js b/apps/web/next.config.js index 9bfe4a0..bdadcc2 100644 --- a/apps/web/next.config.js +++ b/apps/web/next.config.js @@ -5,6 +5,8 @@ await import("./src/env.js"); /** @type {import("next").NextConfig} */ -const config = {}; +const config = { + output: process.env.DOCKER_OUTPUT ? "standalone" : undefined, +}; export default config; diff --git a/apps/web/package.json b/apps/web/package.json index 5148f00..f950756 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "dev": "next dev -p 3000", - "build": "pnpm db:migrate-deploy && next build", + "build": "next build", "start": "next start", "lint": "eslint . --max-warnings 0", "db:post-install": "prisma generate", @@ -37,6 +37,7 @@ "install": "^0.13.0", "lucide-react": "^0.359.0", "mime-types": "^2.1.35", + "nanoid": "^5.0.7", "next": "^14.2.1", "next-auth": "^4.24.6", "pg-boss": "^9.0.3", diff --git a/apps/web/prisma/migrations/20240610073426_add_ses_settings/migration.sql b/apps/web/prisma/migrations/20240610073426_add_ses_settings/migration.sql new file mode 100644 index 0000000..6869687 --- /dev/null +++ b/apps/web/prisma/migrations/20240610073426_add_ses_settings/migration.sql @@ -0,0 +1,25 @@ +-- CreateTable +CREATE TABLE "SesSetting" ( + "id" TEXT NOT NULL, + "region" TEXT NOT NULL, + "idPrefix" TEXT NOT NULL, + "topic" TEXT NOT NULL, + "topicArn" TEXT, + "callbackUrl" TEXT NOT NULL, + "callbackSuccess" BOOLEAN NOT NULL DEFAULT false, + "configGeneral" TEXT, + "configGeneralSuccess" BOOLEAN NOT NULL DEFAULT false, + "configClick" TEXT, + "configClickSuccess" BOOLEAN NOT NULL DEFAULT false, + "configOpen" TEXT, + "configOpenSuccess" BOOLEAN NOT NULL DEFAULT false, + "configFull" TEXT, + "configFullSuccess" BOOLEAN NOT NULL DEFAULT false, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "SesSetting_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "SesSetting_region_key" ON "SesSetting"("region"); diff --git a/apps/web/prisma/schema.prisma b/apps/web/prisma/schema.prisma index db145dc..4453a0a 100644 --- a/apps/web/prisma/schema.prisma +++ b/apps/web/prisma/schema.prisma @@ -19,6 +19,26 @@ model AppSetting { value String } +model SesSetting { + id String @id @default(cuid()) + region String @unique + idPrefix String + topic String + topicArn String? + callbackUrl String + callbackSuccess Boolean @default(false) + configGeneral String? + configGeneralSuccess Boolean @default(false) + configClick String? + configClickSuccess Boolean @default(false) + configOpen String? + configOpenSuccess Boolean @default(false) + configFull String? + configFullSuccess Boolean @default(false) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + // Necessary for Next auth model Account { id String @id @default(cuid()) diff --git a/apps/web/src/app/(dashboard)/api-keys/api-list.tsx b/apps/web/src/app/(dashboard)/api-keys/api-list.tsx index a2c333e..cadd290 100644 --- a/apps/web/src/app/(dashboard)/api-keys/api-list.tsx +++ b/apps/web/src/app/(dashboard)/api-keys/api-list.tsx @@ -40,6 +40,12 @@ export default function ApiList() { /> + ) : apiKeysQuery.data?.length === 0 ? ( + + +

No API keys added

+
+
) : ( apiKeysQuery.data?.map((apiKey) => ( diff --git a/apps/web/src/app/(dashboard)/layout.tsx b/apps/web/src/app/(dashboard)/layout.tsx index 01c3c44..3229567 100644 --- a/apps/web/src/app/(dashboard)/layout.tsx +++ b/apps/web/src/app/(dashboard)/layout.tsx @@ -8,6 +8,7 @@ import { Home, LayoutDashboard, LineChart, + LogOut, Mail, Menu, Package, @@ -28,7 +29,7 @@ import { } from "@unsend/ui/src/dropdown-menu"; import { Sheet, SheetContent, SheetTrigger } from "@unsend/ui/src/sheet"; -import { NavButton } from "./nav-button"; +import { LogoutButton, NavButton } from "./nav-button"; import { DashboardProvider } from "~/providers/dashboard-provider"; import { NextAuthProvider } from "~/providers/next-auth"; @@ -89,15 +90,16 @@ export default function AuthenticatedDashboardLayout({ Developer settings -
+
Docs +
diff --git a/apps/web/src/app/(dashboard)/nav-button.tsx b/apps/web/src/app/(dashboard)/nav-button.tsx index fadbc88..ebc1b6c 100644 --- a/apps/web/src/app/(dashboard)/nav-button.tsx +++ b/apps/web/src/app/(dashboard)/nav-button.tsx @@ -1,5 +1,7 @@ "use client"; +import { LogOut } from "lucide-react"; +import { signOut } from "next-auth/react"; import Link from "next/link"; import { usePathname } from "next/navigation"; import React from "react"; @@ -37,3 +39,15 @@ export const NavButton: React.FC<{ ); }; + +export const LogoutButton: React.FC = () => { + return ( + + ); +}; diff --git a/apps/web/src/app/api/ses_callback/route.ts b/apps/web/src/app/api/ses_callback/route.ts index 5e1e468..7169dd7 100644 --- a/apps/web/src/app/api/ses_callback/route.ts +++ b/apps/web/src/app/api/ses_callback/route.ts @@ -1,3 +1,4 @@ +import { db } from "~/server/db"; import { AppSettingsService } from "~/server/service/app-settings-service"; import { parseSesHook } from "~/server/service/ses-hook-parser"; import { SnsNotificationMessage } from "~/types/aws-types"; @@ -13,6 +14,10 @@ export async function POST(req: Request) { console.log(data, data.Message); + if (isFromUnsend(data)) { + return Response.json({ data: "success" }); + } + const isEventValid = await checkEventValidity(data); console.log("isEventValid: ", isEventValid); @@ -47,9 +52,38 @@ async function handleSubscription(message: any) { method: "GET", }); + const topicArn = message.TopicArn as string; + const setting = await db.sesSetting.findFirst({ + where: { + topicArn, + }, + }); + + if (!setting) { + return Response.json({ data: "Setting not found" }); + } + + await db.sesSetting.update({ + where: { + id: setting?.id, + }, + data: { + callbackSuccess: true, + }, + }); + return Response.json({ data: "Success" }); } +// A simple check to ensure that the event is from the correct topic +function isFromUnsend({ fromUnsend }: { fromUnsend: boolean }) { + if (fromUnsend) { + return true; + } + + return false; +} + // A simple check to ensure that the event is from the correct topic async function checkEventValidity(message: SnsNotificationMessage) { const { TopicArn } = message; diff --git a/apps/web/src/app/layout.tsx b/apps/web/src/app/layout.tsx index e215804..2f09f2e 100644 --- a/apps/web/src/app/layout.tsx +++ b/apps/web/src/app/layout.tsx @@ -6,6 +6,7 @@ import { Toaster } from "@unsend/ui/src/toaster"; import { TRPCReactProvider } from "~/trpc/react"; import { Metadata } from "next"; +import { getBoss } from "~/server/service/job-service"; const inter = Inter({ subsets: ["latin"], @@ -18,11 +19,17 @@ export const metadata: Metadata = { icons: [{ rel: "icon", url: "/favicon.ico" }], }; -export default function RootLayout({ +export default async function RootLayout({ children, }: { children: React.ReactNode; }) { + /** + * Because I don't know a better way to call this during server startup. + * This is a temporary fix to ensure that the boss is running. + */ + await getBoss(); + return ( diff --git a/apps/web/src/app/login/login-page.tsx b/apps/web/src/app/login/login-page.tsx index b91bc24..01f6536 100644 --- a/apps/web/src/app/login/login-page.tsx +++ b/apps/web/src/app/login/login-page.tsx @@ -22,6 +22,7 @@ import { REGEXP_ONLY_DIGITS_AND_CHARS, } from "@unsend/ui/src/input-otp"; import { Input } from "@unsend/ui/src/input"; +import { env } from "~/env"; const emailSchema = z.object({ email: z @@ -93,108 +94,129 @@ export default function LoginPage() { Continue with Github - -
-

- or -

-
-
- {emailStatus === "success" ? ( + + + + Continue with Google + + ) : null} + {env.NEXT_PUBLIC_IS_CLOUD ? ( <> -

- We have sent an email with the OTP. Please check your inbox -

-
- - ( - - - - - - - - - - - - +
+

+ or +

+
+
+ {emailStatus === "success" ? ( + <> +

+ We have sent an email with the OTP. Please check your inbox +

+ + + ( + + + + + + + + + + + + - - - )} - /> + +
+ )} + /> - - - + + + + + ) : ( + <> +
+ + ( + + + + + + + + )} + /> + + + + + )} - ) : ( - <> -
- - ( - - - - - - - - )} - /> - - - - - )} + ) : null} diff --git a/apps/web/src/env.js b/apps/web/src/env.js index 3e19c91..8d502b6 100644 --- a/apps/web/src/env.js +++ b/apps/web/src/env.js @@ -34,16 +34,18 @@ export const env = createEnv({ AWS_SECRET_KEY: z.string(), APP_URL: z.string().optional(), SNS_TOPIC: z.string(), - UNSEND_API_KEY: z.string(), - UNSEND_URL: z.string(), - GOOGLE_CLIENT_ID: z.string(), - GOOGLE_CLIENT_SECRET: z.string(), + UNSEND_API_KEY: z.string().optional(), + UNSEND_URL: z.string().optional(), + GOOGLE_CLIENT_ID: z.string().optional(), + GOOGLE_CLIENT_SECRET: z.string().optional(), SES_QUEUE_LIMIT: z.string().transform((str) => parseInt(str, 10)), AWS_DEFAULT_REGION: z.string().default("us-east-1"), API_RATE_LIMIT: z .string() .transform((str) => parseInt(str, 10)) .default(2), + FROM_EMAIL: z.string().optional(), + ADMIN_EMAIL: z.string().optional(), }, /** @@ -53,6 +55,7 @@ export const env = createEnv({ */ client: { // NEXT_PUBLIC_CLIENTVAR: z.string(), + NEXT_PUBLIC_IS_CLOUD: z.string().transform((str) => str === "true"), }, /** @@ -77,12 +80,14 @@ export const env = createEnv({ SES_QUEUE_LIMIT: process.env.SES_QUEUE_LIMIT, AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION, API_RATE_LIMIT: process.env.API_RATE_LIMIT, + NEXT_PUBLIC_IS_CLOUD: process.env.NEXT_PUBLIC_IS_CLOUD, + ADMIN_EMAIL: process.env.ADMIN_EMAIL, }, /** * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation. This is especially * useful for Docker builds. */ - skipValidation: !!process.env.SKIP_ENV_VALIDATION, + skipValidation: process.env.SKIP_ENV_VALIDATION === "true", /** * Makes it so that empty strings are treated as undefined. `SOME_VAR: z.string()` and * `SOME_VAR=''` will throw an error. diff --git a/apps/web/src/server/api/routers/admin.ts b/apps/web/src/server/api/routers/admin.ts new file mode 100644 index 0000000..4ee4a08 --- /dev/null +++ b/apps/web/src/server/api/routers/admin.ts @@ -0,0 +1,37 @@ +import { z } from "zod"; +import { env } from "~/env"; + +import { createTRPCRouter, adminProcedure } from "~/server/api/trpc"; +import { SesSettingsService } from "~/server/service/ses-settings-service"; + +export const adminRouter = createTRPCRouter({ + getSesSettings: adminProcedure.query(async () => { + return SesSettingsService.getAllSettings(); + }), + + addSesSettings: adminProcedure + .input( + z.object({ + region: z.string(), + unsendUrl: z.string().url(), + }) + ) + .mutation(async ({ input }) => { + return SesSettingsService.createSesSetting({ + region: input.region, + unsendUrl: input.unsendUrl, + }); + }), + + getSetting: adminProcedure + .input( + z.object({ + region: z.string().optional().nullable(), + }) + ) + .query(async ({ input }) => { + return SesSettingsService.getSetting( + input.region ?? env.AWS_DEFAULT_REGION + ); + }), +}); diff --git a/apps/web/src/server/api/trpc.ts b/apps/web/src/server/api/trpc.ts index 51a1097..8f022be 100644 --- a/apps/web/src/server/api/trpc.ts +++ b/apps/web/src/server/api/trpc.ts @@ -10,6 +10,7 @@ import { initTRPC, TRPCError } from "@trpc/server"; import superjson from "superjson"; import { ZodError } from "zod"; +import { env } from "~/env"; import { getServerAuthSession } from "~/server/auth"; import { db } from "~/server/db"; @@ -123,3 +124,13 @@ export const teamProcedure = protectedProcedure.use(async ({ ctx, next }) => { }, }); }); + +/** + * To manage application settings, for hosted version, authenticated users will be considered as admin + */ +export const adminProcedure = protectedProcedure.use(async ({ ctx, next }) => { + if (env.NEXT_PUBLIC_IS_CLOUD && ctx.session.user.email !== env.ADMIN_EMAIL) { + throw new TRPCError({ code: "UNAUTHORIZED" }); + } + return next(); +}); diff --git a/apps/web/src/server/auth.ts b/apps/web/src/server/auth.ts index 7292f2d..b297c42 100644 --- a/apps/web/src/server/auth.ts +++ b/apps/web/src/server/auth.ts @@ -1,5 +1,6 @@ import { PrismaAdapter } from "@auth/prisma-adapter"; import { + AuthOptions, getServerSession, type DefaultSession, type NextAuthOptions, @@ -36,6 +37,42 @@ declare module "next-auth" { } } +/** + * Auth providers + */ + +const providers: Provider[] = [ + GitHubProvider({ + clientId: env.GITHUB_ID, + clientSecret: env.GITHUB_SECRET, + allowDangerousEmailAccountLinking: true, + }), +]; + +if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) { + providers.push( + GoogleProvider({ + clientId: env.GOOGLE_CLIENT_ID, + clientSecret: env.GOOGLE_CLIENT_SECRET, + allowDangerousEmailAccountLinking: true, + }) + ); +} + +if (env.FROM_EMAIL) { + providers.push( + EmailProvider({ + from: env.FROM_EMAIL, + async sendVerificationRequest({ identifier: email, url, token }) { + await sendSignUpEmail(email, token, url); + }, + async generateVerificationToken() { + return Math.random().toString(36).substring(2, 7).toLowerCase(); + }, + }) + ); +} + /** * Options for NextAuth.js used to configure adapters, providers, callbacks, etc. * @@ -56,36 +93,18 @@ export const authOptions: NextAuthOptions = { pages: { signIn: "/login", }, - providers: [ - /** - * ...add more providers here. - * - * Most other providers require a bit more work than the Discord provider. For example, the - * GitHub provider requires you to add the `refresh_token_expires_in` field to the Account - * model. Refer to the NextAuth.js docs for the provider you want to use. Example: - * - * @see https://next-auth.js.org/providers/github - */ - GitHubProvider({ - clientId: env.GITHUB_ID, - clientSecret: env.GITHUB_SECRET, - allowDangerousEmailAccountLinking: true, - }), - GoogleProvider({ - clientId: env.GOOGLE_CLIENT_ID, - clientSecret: env.GOOGLE_CLIENT_SECRET, - allowDangerousEmailAccountLinking: true, - }), - EmailProvider({ - from: "no-reply@splitpro.app", - async sendVerificationRequest({ identifier: email, url, token }) { - await sendSignUpEmail(email, token, url); - }, - async generateVerificationToken() { - return Math.random().toString(36).substring(2, 7).toLowerCase(); - }, - }), - ], + events: { + createUser: async ({ user }) => { + // No waitlist for self hosting + if (!env.NEXT_PUBLIC_IS_CLOUD) { + await db.user.update({ + where: { id: user.id }, + data: { isBetaUser: true }, + }); + } + }, + }, + providers, }; /** @@ -97,6 +116,7 @@ export const getServerAuthSession = () => getServerSession(authOptions); import { createHash } from "crypto"; import { sendSignUpEmail } from "./mailer"; +import { Provider } from "next-auth/providers/index"; /** * Hashes a token using SHA-256. diff --git a/apps/web/src/server/mailer.ts b/apps/web/src/server/mailer.ts index 904c10f..b5b5a7c 100644 --- a/apps/web/src/server/mailer.ts +++ b/apps/web/src/server/mailer.ts @@ -1,7 +1,14 @@ import { env } from "~/env"; import { Unsend } from "unsend"; -const unsend = new Unsend(env.UNSEND_API_KEY); +let unsend: Unsend | undefined; + +const getClient = () => { + if (!unsend) { + unsend = new Unsend(env.UNSEND_API_KEY); + } + return unsend; +}; export async function sendSignUpEmail( email: string, @@ -28,10 +35,10 @@ async function sendMail( text: string, html: string ) { - if (env.UNSEND_API_KEY && env.UNSEND_URL) { - const resp = await unsend.emails.send({ + if (env.UNSEND_API_KEY && env.UNSEND_URL && env.FROM_EMAIL) { + const resp = await getClient().emails.send({ to: email, - from: "no-reply@auth.unsend.dev", + from: env.FROM_EMAIL, subject, text, html, diff --git a/apps/web/src/server/public-api/auth.ts b/apps/web/src/server/public-api/auth.ts index 8b7b941..69567c3 100644 --- a/apps/web/src/server/public-api/auth.ts +++ b/apps/web/src/server/public-api/auth.ts @@ -10,6 +10,8 @@ const rateLimitCache = new TTLCache({ max: env.API_RATE_LIMIT, }); +console.log(env.DATABASE_URL); + /** * Gets the team from the token. Also will check if the token is valid. */ diff --git a/apps/web/src/server/service/job-service.ts b/apps/web/src/server/service/job-service.ts index 7940aab..2c81174 100644 --- a/apps/web/src/server/service/job-service.ts +++ b/apps/web/src/server/service/job-service.ts @@ -15,7 +15,7 @@ const boss = new pgBoss({ }); let started = false; -async function getBoss() { +export async function getBoss() { if (!started) { await boss.start(); await boss.work( diff --git a/apps/web/src/server/service/ses-settings-service.ts b/apps/web/src/server/service/ses-settings-service.ts new file mode 100644 index 0000000..2090214 --- /dev/null +++ b/apps/web/src/server/service/ses-settings-service.ts @@ -0,0 +1,195 @@ +import { SesSetting } from "@prisma/client"; +import { db } from "../db"; +import { env } from "~/env"; +import { customAlphabet } from "nanoid"; +import * as sns from "~/server/aws/sns"; +import * as ses from "~/server/aws/ses"; +import { EventType } from "@aws-sdk/client-sesv2"; + +const nanoid = customAlphabet("1234567890abcdef", 10); + +const GENERAL_EVENTS: EventType[] = [ + "BOUNCE", + "COMPLAINT", + "DELIVERY", + "DELIVERY_DELAY", + "REJECT", + "RENDERING_FAILURE", + "SEND", + "SUBSCRIPTION", +]; + +export class SesSettingsService { + private static cache: Record = {}; + + public static getSetting(region = env.AWS_DEFAULT_REGION): SesSetting | null { + if (this.cache[region]) { + return this.cache[region] as SesSetting; + } + return null; + } + + public static getAllSettings() { + return Object.values(this.cache); + } + + /** + * Creates a new setting in AWS for the given region and unsendUrl + * + * @param region + * @param unsendUrl + */ + public static async createSesSetting({ + region, + unsendUrl, + }: { + region: string; + unsendUrl: string; + }) { + if (this.cache[region]) { + throw new Error(`SesSetting for region ${region} already exists`); + } + + const unsendUrlValidation = await isValidUnsendUrl(unsendUrl); + + if (!unsendUrlValidation.isValid) { + throw new Error( + `Unsend URL ${unsendUrl} is not valid, status: ${unsendUrlValidation.code} ${unsendUrlValidation.error}` + ); + } + + const idPrefix = nanoid(10); + + const setting = await db.sesSetting.create({ + data: { + region, + callbackUrl: `${unsendUrl}/api/ses_callback`, + topic: `${idPrefix}-${region}-unsend`, + idPrefix, + }, + }); + + await createSettingInAws(setting); + + this.invalidateCache(); + } + + public static async init() { + const settings = await db.sesSetting.findMany(); + settings.forEach((setting) => { + this.cache[setting.region] = setting; + }); + } + + static invalidateCache() { + this.cache = {}; + this.init(); + } +} + +async function createSettingInAws(setting: SesSetting) { + await registerTopicInAws(setting).then(registerConfigurationSet); +} + +/** + * Creates a new topic in AWS and subscribes the callback URL to it + */ +async function registerTopicInAws(setting: SesSetting) { + const topicArn = await sns.createTopic(setting.topic); + + if (!topicArn) { + throw new Error("Failed to create SNS topic"); + } + + await sns.subscribeEndpoint( + topicArn, + `${setting.callbackUrl}/api/ses_callback` + ); + + return await db.sesSetting.update({ + where: { + id: setting.id, + }, + data: { + topicArn, + }, + }); +} + +/** + * Creates a new configuration set in AWS for given region + * Totally consist of 4 configs. + * 1. General - for general events + * 2. Click - for click tracking + * 3. Open - for open tracking + * 4. Full - for click and open tracking + */ +async function registerConfigurationSet(setting: SesSetting) { + if (!setting.topicArn) { + throw new Error("Setting does not have a topic ARN"); + } + + const configGeneral = `${setting.idPrefix}-${setting.region}-unsend-general`; + const generalStatus = await ses.addWebhookConfiguration( + configGeneral, + setting.topicArn, + GENERAL_EVENTS + ); + + const configClick = `${setting.idPrefix}-${setting.region}-unsend-click`; + const clickStatus = await ses.addWebhookConfiguration( + configClick, + setting.topicArn, + [...GENERAL_EVENTS, "CLICK"] + ); + + const configOpen = `${setting.idPrefix}-${setting.region}-unsend-open`; + const openStatus = await ses.addWebhookConfiguration( + configOpen, + setting.topicArn, + [...GENERAL_EVENTS, "OPEN"] + ); + + const configFull = `${setting.idPrefix}-${setting.region}-unsend-full`; + const fullStatus = await ses.addWebhookConfiguration( + configFull, + setting.topicArn, + [...GENERAL_EVENTS, "CLICK", "OPEN"] + ); + + return await db.sesSetting.update({ + where: { + id: setting.id, + }, + data: { + configGeneral, + configGeneralSuccess: generalStatus, + configClick, + configClickSuccess: clickStatus, + configOpen, + configOpenSuccess: openStatus, + configFull, + configFullSuccess: fullStatus, + }, + }); +} + +async function isValidUnsendUrl(url: string) { + try { + const response = await fetch(`${url}/api/ses_callback`, { + method: "POST", + body: JSON.stringify({ fromUnsend: true }), + }); + return { + isValid: response.status === 200, + code: response.status, + error: response.statusText, + }; + } catch (e) { + return { + isValid: false, + code: 500, + error: e, + }; + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7306dff --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +name: unsend-prod + +services: + postgres: + image: postgres:16 + container_name: postgres + restart: always + environment: + - POSTGRES_USER=${POSTGRES_USER:?err} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?err} + - POSTGRES_DB=${POSTGRES_DB:?err} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - database:/var/lib/postgresql/data + + # You don't need to expose this port to the host since, docker compose creates an internal network + # through which both of these containers could talk to each other using their container_name as hostname + # But if you want to connect this to a querying tool to debug you can definitely uncomment this + # ports: + # - "5432:5432" + + unsend: + build: + dockerfile: Dockerfile + image: unsend + container_name: unsend + restart: always + ports: + - ${PORT:-3000}:${PORT:-3000} + environment: + - PORT=${PORT:-3000} + - DATABASE_URL=${DATABASE_URL:?err} + - NEXTAUTH_URL=${NEXTAUTH_URL:?err} + - NEXTAUTH_SECRET=${NEXTAUTH_SECRET:?err} + - AWS_ACCESS_KEY=${AWS_ACCESS_KEY:?err} + - AWS_SECRET_KEY=${AWS_SECRET_KEY:?err} + - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:?err} + - GITHUB_ID=${GITHUB_ID:?err} + - GITHUB_SECRET=${GITHUB_SECRET:?err} + - APP_URL=${APP_URL:-${NEXTAUTH_URL}} + - SNS_TOPIC=${SNS_TOPIC:?err} + - NEXT_PUBLIC_IS_CLOUD=${NEXT_PUBLIC_IS_CLOUD:-false} + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - SES_QUEUE_LIMIT=${SES_QUEUE_LIMIT:-1} + - API_RATE_LIMIT=${API_RATE_LIMIT:-1} + depends_on: + postgres: + condition: service_healthy + +volumes: + database: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5185538..bb08aa4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -154,6 +154,9 @@ importers: mime-types: specifier: ^2.1.35 version: 2.1.35 + nanoid: + specifier: ^5.0.7 + version: 5.0.7 next: specifier: ^14.2.1 version: 14.2.1(react-dom@18.2.0)(react@18.2.0) @@ -9036,6 +9039,12 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + /nanoid@5.0.7: + resolution: {integrity: sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==} + engines: {node: ^18 || >=20} + hasBin: true + dev: false + /natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..ec5de9b --- /dev/null +++ b/start.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +echo "Deploying prisma migrations" + +pnpx prisma migrate deploy --schema ./apps/web/prisma/schema.prisma + +echo "Starting web server" + +node apps/web/server.js + diff --git a/turbo.json b/turbo.json index 6ee05ab..0ed3b49 100644 --- a/turbo.json +++ b/turbo.json @@ -34,7 +34,8 @@ "UNSEND_API_KEY", "UNSEND_URL", "GOOGLE_CLIENT_ID", - "GOOGLE_CLIENT_SECRET" + "GOOGLE_CLIENT_SECRET", + "NEXT_PUBLIC_IS_CLOUD" ] }, "lint": {