feat: add daily email usage (#97)

* add daily email usage

* remove console

apps/web/prisma/migrations/20250201051912_add_indexes/migration.sql

@@ -0,0 +1,8 @@
+-- CreateIndex
+CREATE INDEX "Campaign_createdAt_idx" ON "Campaign"("createdAt" DESC);
+
+-- CreateIndex
+CREATE INDEX "Email_createdAt_idx" ON "Email"("createdAt" DESC);
+
+-- CreateIndex
+CREATE INDEX "EmailEvent_emailId_idx" ON "EmailEvent"("emailId");

apps/web/prisma/migrations/20250201131024_add_daily_usage/migration.sql

@@ -0,0 +1,22 @@
+-- CreateEnum
+CREATE TYPE "EmailUsageType" AS ENUM ('TRANSACTIONAL', 'MARKETING');
+
+-- CreateTable
+CREATE TABLE "DailyEmailUsage" (
+    "teamId" INTEGER NOT NULL,
+    "date" TEXT NOT NULL,
+    "type" "EmailUsageType" NOT NULL,
+    "domainId" INTEGER NOT NULL,
+    "delivered" INTEGER NOT NULL,
+    "opened" INTEGER NOT NULL,
+    "clicked" INTEGER NOT NULL,
+    "bounced" INTEGER NOT NULL,
+    "complained" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "DailyEmailUsage_pkey" PRIMARY KEY ("teamId","domainId","date","type")
+);
+
+-- AddForeignKey
+ALTER TABLE "DailyEmailUsage" ADD CONSTRAINT "DailyEmailUsage_teamId_fkey" FOREIGN KEY ("teamId") REFERENCES "Team"("id") ON DELETE CASCADE ON UPDATE CASCADE;

apps/web/prisma/migrations/20250201131959_add_sent_to_daily_usage/migration.sql

@@ -0,0 +1,7 @@
+-- AlterTable
+ALTER TABLE "DailyEmailUsage" ADD COLUMN     "sent" INTEGER NOT NULL DEFAULT 0,
+ALTER COLUMN "delivered" SET DEFAULT 0,
+ALTER COLUMN "opened" SET DEFAULT 0,
+ALTER COLUMN "clicked" SET DEFAULT 0,
+ALTER COLUMN "bounced" SET DEFAULT 0,
+ALTER COLUMN "complained" SET DEFAULT 0;

apps/web/prisma/schema.prisma

@@ -91,16 +91,17 @@ model User {
 }
 
 model Team {
-  id           Int           @id @default(autoincrement())
+  id               Int               @id @default(autoincrement())
-  name         String
+  name             String
-  createdAt    DateTime      @default(now())
+  createdAt        DateTime          @default(now())
-  updatedAt    DateTime      @updatedAt
+  updatedAt        DateTime          @updatedAt
-  teamUsers    TeamUser[]
+  teamUsers        TeamUser[]
-  domains      Domain[]
+  domains          Domain[]
-  apiKeys      ApiKey[]
+  apiKeys          ApiKey[]
-  emails       Email[]
+  emails           Email[]
-  contactBooks ContactBook[]
+  contactBooks     ContactBook[]
-  campaigns    Campaign[]
+  campaigns        Campaign[]
+  dailyEmailUsages DailyEmailUsage[]
 }
 
 enum Role {
@@ -203,6 +204,8 @@ model Email {
   contactId    String?
   team         Team         @relation(fields: [teamId], references: [id], onDelete: Cascade)
   emailEvents  EmailEvent[]
+
+  @@index([createdAt(sort: Desc)])
 }
 
 model EmailEvent {
@@ -212,6 +215,8 @@ model EmailEvent {
   data      Json?
   createdAt DateTime    @default(now())
   email     Email       @relation(fields: [emailId], references: [id], onDelete: Cascade)
+
+  @@index([emailId])
 }
 
 model ContactBook {
@@ -277,4 +282,30 @@ model Campaign {
   updatedAt     DateTime       @updatedAt
 
   team Team @relation(fields: [teamId], references: [id], onDelete: Cascade)
+
+  @@index([createdAt(sort: Desc)])
+}
+
+enum EmailUsageType {
+  TRANSACTIONAL
+  MARKETING
+}
+
+model DailyEmailUsage {
+  teamId     Int
+  date       String
+  type       EmailUsageType
+  domainId   Int
+  sent       Int            @default(0)
+  delivered  Int            @default(0)
+  opened     Int            @default(0)
+  clicked    Int            @default(0)
+  bounced    Int            @default(0)
+  complained Int            @default(0)
+  createdAt  DateTime       @default(now())
+  updatedAt  DateTime       @updatedAt
+
+  team Team @relation(fields: [teamId], references: [id], onDelete: Cascade)
+
+  @@id([teamId, domainId, date, type])
 }

apps/web/src/app/(dashboard)/dashboard/dashboard-chart.tsx

@@ -43,42 +43,47 @@ export default function DashboardChart() {
             <>
               <DashboardItemCard
                 status={"total"}
-                count={statusQuery.data.totalCount}
+                count={statusQuery.data.totalCounts.sent}
                 percentage={100}
               />
               <DashboardItemCard
                 status={EmailStatus.DELIVERED}
-                count={statusQuery.data.emailStatusCounts.DELIVERED.count}
+                count={statusQuery.data.totalCounts.delivered}
                 percentage={
-                  statusQuery.data.emailStatusCounts.DELIVERED.percentage
+                  statusQuery.data.totalCounts.delivered /
+                  statusQuery.data.totalCounts.sent
                 }
               />
               <DashboardItemCard
                 status={EmailStatus.BOUNCED}
-                count={statusQuery.data.emailStatusCounts.BOUNCED.count}
+                count={statusQuery.data.totalCounts.bounced}
                 percentage={
-                  statusQuery.data.emailStatusCounts.BOUNCED.percentage
+                  statusQuery.data.totalCounts.bounced /
+                  statusQuery.data.totalCounts.sent
                 }
               />
               <DashboardItemCard
                 status={EmailStatus.COMPLAINED}
-                count={statusQuery.data.emailStatusCounts.COMPLAINED.count}
+                count={statusQuery.data.totalCounts.complained}
                 percentage={
-                  statusQuery.data.emailStatusCounts.COMPLAINED.percentage
+                  statusQuery.data.totalCounts.complained /
+                  statusQuery.data.totalCounts.sent
                 }
               />
               <DashboardItemCard
                 status={EmailStatus.CLICKED}
-                count={statusQuery.data.emailStatusCounts.CLICKED.count}
+                count={statusQuery.data.totalCounts.clicked}
                 percentage={
-                  statusQuery.data.emailStatusCounts.CLICKED.percentage
+                  statusQuery.data.totalCounts.clicked /
+                  statusQuery.data.totalCounts.sent
                 }
               />
               <DashboardItemCard
                 status={EmailStatus.OPENED}
-                count={statusQuery.data.emailStatusCounts.OPENED.count}
+                count={statusQuery.data.totalCounts.opened}
                 percentage={
-                  statusQuery.data.emailStatusCounts.OPENED.percentage
+                  statusQuery.data.totalCounts.opened /
+                  statusQuery.data.totalCounts.sent
                 }
               />
             </>
@@ -108,7 +113,7 @@ export default function DashboardChart() {
               <BarChart
                 width={900}
                 height={300}
-                data={statusQuery.data.emailDailyStatusCounts}
+                data={statusQuery.data.result}
                 margin={{
                   top: 20,
                   right: 30,
@@ -117,77 +122,74 @@ export default function DashboardChart() {
                 }}
               >
                 <CartesianGrid vertical={false} strokeDasharray="3 3" />
-                <XAxis dataKey="name" fontSize={12} />
+                <XAxis dataKey="date" fontSize={12} />
                 <YAxis fontSize={12} />
                 <Tooltip
                   content={({ payload }) => {
                     const data = payload?.[0]?.payload as Record<
-                      EmailStatus,
+                      | "sent"
+                      | "delivered"
+                      | "opened"
+                      | "clicked"
+                      | "bounced"
+                      | "complained",
                       number
-                    > & { name: string };
+                    > & { date: string };
 
-                    if (
+                    if (!data || data.sent === 0) return null;
-                      !data ||
-                      (!data.BOUNCED &&
-                        !data.COMPLAINED &&
-                        !data.DELIVERED &&
-                        !data.OPENED &&
-                        !data.CLICKED)
-                    )
-                      return null;
 
                     return (
                       <div className=" bg-background border shadow-lg p-2 rounded flex flex-col gap-2 px-4">
                         <p className="text-sm text-muted-foreground">
-                          {data.name}
+                          {data.date}
                         </p>
-                        {data.DELIVERED ? (
+                        {data.delivered ? (
                           <div className="flex gap-2 items-center">
                             <div className="w-2.5 h-2.5 bg-[#10b981] rounded-[2px]"></div>
                             <p className="text-xs text-muted-foreground w-[60px]">
                               Delivered
                             </p>
                             <p className="text-xs font-mono">
-                              {data.DELIVERED}
+                              {data.delivered}
                             </p>
                           </div>
                         ) : null}
-                        {data.BOUNCED ? (
+                        {data.bounced ? (
                           <div className="flex gap-2 items-center">
                             <div className="w-2.5 h-2.5 bg-[#ef4444] rounded-[2px]"></div>
                             <p className="text-xs text-muted-foreground w-[60px]">
                               Bounced
                             </p>
-                            <p className="text-xs font-mono">{data.BOUNCED}</p>
+                            <p className="text-xs font-mono">{data.bounced}</p>
                           </div>
                         ) : null}
-                        {data.COMPLAINED ? (
+                        {data.complained ? (
                           <div className="flex gap-2 items-center">
                             <div className="w-2.5 h-2.5 bg-[#eab308] rounded-[2px]"></div>
                             <p className="text-xs text-muted-foreground w-[60px]">
                               Complained
                             </p>
                             <p className="text-xs font-mono">
-                              {data.COMPLAINED}
+                              {data.complained}
                             </p>
                           </div>
                         ) : null}
-                        {data.OPENED ? (
+                        {data.opened ? (
                           <div className="flex gap-2 items-center">
                             <div className="w-2.5 h-2.5 bg-[#6366f1] rounded-[2px]"></div>
                             <p className="text-xs text-muted-foreground w-[60px]">
                               Opened
                             </p>
-                            <p className="text-xs font-mono">{data.OPENED}</p>
+                            <p className="text-xs font-mono">{data.opened}</p>
                           </div>
                         ) : null}
-                        {data.CLICKED ? (
+                        {data.clicked ? (
                           <div className="flex gap-2 items-center">
                             <div className="w-2.5 h-2.5 bg-[#06b6d4] rounded-[2px]"></div>
                             <p className="text-xs text-muted-foreground w-[60px]">
                               Clicked
                             </p>
-                            <p className="text-xs font-mono">{data.CLICKED}</p>
+                            <p className="text-xs font-mono">{data.clicked}</p>
                           </div>
                         ) : null}
                       </div>
@@ -198,14 +200,14 @@ export default function DashboardChart() {
                 {/* <Legend /> */}
                 <Bar
                   barSize={8}
-                  dataKey="DELIVERED"
+                  dataKey="delivered"
                   stackId="a"
                   fill="#10b981"
                 />
-                <Bar dataKey="BOUNCED" stackId="a" fill="#ef4444" />
+                <Bar dataKey="bounced" stackId="a" fill="#ef4444" />
-                <Bar dataKey="COMPLAINED" stackId="a" fill="#eab308" />
+                <Bar dataKey="complained" stackId="a" fill="#eab308" />
-                <Bar dataKey="OPENED" stackId="a" fill="#6366f1" />
+                <Bar dataKey="opened" stackId="a" fill="#6366f1" />
-                <Bar dataKey="CLICKED" stackId="a" fill="#06b6d4" />
+                <Bar dataKey="clicked" stackId="a" fill="#06b6d4" />
               </BarChart>
             </ResponsiveContainer>
           </div>
@@ -237,7 +239,7 @@ const DashboardItemCard: React.FC<DashboardItemCardProps> = ({
           {count}
         </div>
         {status !== "total" ? (
-          <div className="text-sm pb-1">{percentage}%</div>
+          <div className="text-sm pb-1">{(percentage * 100).toFixed(0)}%</div>
         ) : null}
       </div>
     </div>

apps/web/src/app/api/ses_callback/route.ts

@@ -1,5 +1,5 @@
 import { db } from "~/server/db";
-import { parseSesHook } from "~/server/service/ses-hook-parser";
+import { parseSesHook, SesHookParser } from "~/server/service/ses-hook-parser";
 import { SesSettingsService } from "~/server/service/ses-settings-service";
 import { SnsNotificationMessage } from "~/types/aws-types";
 
@@ -16,7 +16,7 @@ export async function POST(req: Request) {
 
   const isEventValid = await checkEventValidity(data);
 
-  console.log("isEventValid: ", isEventValid);
+  console.log("Is event valid: ", isEventValid);
 
   if (!isEventValid) {
     return Response.json({ data: "Event is not valid" });
@@ -30,7 +30,10 @@ export async function POST(req: Request) {
 
   try {
     message = JSON.parse(data.Message || "{}");
-    const status = await parseSesHook(message);
+    const status = await SesHookParser.queue({
+      event: message,
+      messageId: data.MessageId,
+    });
     console.log("Error is parsing hook", !status);
     if (!status) {
       return Response.json({ data: "Error is parsing hook" });
@@ -43,6 +46,9 @@ export async function POST(req: Request) {
   }
 }
 
+/**
+ * Handles the subscription confirmation event. called only once for a webhook
+ */
 async function handleSubscription(message: any) {
   await fetch(message.SubscribeURL, {
     method: "GET",
@@ -73,7 +79,9 @@ async function handleSubscription(message: any) {
   return Response.json({ data: "Success" });
 }
 
-// A simple check to ensure that the event is from the correct topic
+/**
+ * A simple check to ensure that the event is from the correct topic
+ */
 async function checkEventValidity(message: SnsNotificationMessage) {
   const { TopicArn } = message;
   const configuredTopicArn = await SesSettingsService.getTopicArns();

apps/web/src/server/api/routers/email.ts

@@ -68,108 +68,85 @@ export const emailRouter = createTRPCRouter({
     .query(async ({ ctx, input }) => {
       const { team } = ctx;
       const days = input.days !== 7 ? 30 : 7;
-      const daysInMs = days * 24 * 60 * 60 * 1000;
 
-      const rawEmailStatusCounts = await db.email.findMany({
+      const startDate = new Date();
-        where: {
+      startDate.setDate(startDate.getDate() - days);
-          teamId: team.id,
+      const isoStartDate = startDate.toISOString().split("T")[0];
-          createdAt: {
-            gt: new Date(Date.now() - daysInMs),
-          },
-        },
-        select: {
-          latestStatus: true,
-          createdAt: true,
-        },
-      });
 
-      const totalCount = rawEmailStatusCounts.length;
+      type DailyEmailUsage = {
+        date: string;
+        sent: number;
+        delivered: number;
+        opened: number;
+        clicked: number;
+        bounced: number;
+        complained: number;
+      };
 
-      const emailStatusCounts = rawEmailStatusCounts.reduce(
+      const result = await db.$queryRaw<Array<DailyEmailUsage>>`
-        (acc, cur) => {
+        SELECT 
-          acc[cur.latestStatus] = {
+          date,
-            count: (acc[cur.latestStatus]?.count || 0) + 1,
+          sent,
-            percentage: Number(
+          delivered,
-              (
+          opened,
-                (((acc[cur.latestStatus]?.count || 0) + 1) / totalCount) *
+          clicked,
-                100
+          bounced,
-              ).toFixed(0)
+          complained
-            ),
+        FROM "DailyEmailUsage"
-          };
+        WHERE "teamId" = ${team.id}
+        AND "date" >= ${isoStartDate}
+        ORDER BY "date" ASC
+      `;
+
+      // Fill in any missing dates with 0 values
+      const filledResult: DailyEmailUsage[] = [];
+      const endDateObj = new Date();
+
+      for (let i = days; i > -1; i--) {
+        const dateStr = subDays(endDateObj, i)
+          .toISOString()
+          .split("T")[0] as string;
+        const existingData = result.find((r) => r.date === dateStr);
+
+        if (existingData) {
+          filledResult.push({
+            ...existingData,
+            date: format(dateStr, "MMM dd"),
+          });
+        } else {
+          filledResult.push({
+            date: format(dateStr, "MMM dd"),
+            sent: 0,
+            delivered: 0,
+            opened: 0,
+            clicked: 0,
+            bounced: 0,
+            complained: 0,
+          });
+        }
+      }
+
+      const totalCounts = result.reduce(
+        (acc, curr) => {
+          acc.sent += curr.sent;
+          acc.delivered += curr.delivered;
+          acc.opened += curr.opened;
+          acc.clicked += curr.clicked;
+          acc.bounced += curr.bounced;
+          acc.complained += curr.complained;
           return acc;
         },
         {
-          DELIVERED: { count: 0, percentage: 0 },
+          sent: 0,
-          COMPLAINED: { count: 0, percentage: 0 },
+          delivered: 0,
-          OPENED: { count: 0, percentage: 0 },
+          opened: 0,
-          CLICKED: { count: 0, percentage: 0 },
+          clicked: 0,
-          BOUNCED: { count: 0, percentage: 0 },
+          bounced: 0,
-        } as Record<EmailStatus, { count: number; percentage: number }>
+          complained: 0,
+        }
       );
 
-      const dateRecord: Record<
+      return { result: filledResult, totalCounts };
-        string,
-        Record<
-          "DELIVERED" | "COMPLAINED" | "OPENED" | "CLICKED" | "BOUNCED",
-          number
-        >
-      > = {};
-
-      const currentDate = new Date();
-
-      for (let i = 0; i < (input.days || 7); i++) {
-        const actualDate = subDays(currentDate, i);
-        dateRecord[format(actualDate, "MMM dd")] = {
-          DELIVERED: 0,
-          COMPLAINED: 0,
-          OPENED: 0,
-          CLICKED: 0,
-          BOUNCED: 0,
-        };
-      }
-
-      const _emailDailyStatusCounts = rawEmailStatusCounts.reduce(
-        (acc, { latestStatus, createdAt }) => {
-          const day = format(createdAt, "MMM dd");
-
-          if (
-            !day ||
-            ![
-              "DELIVERED",
-              "COMPLAINED",
-              "OPENED",
-              "CLICKED",
-              "BOUNCED",
-            ].includes(latestStatus)
-          ) {
-            return acc;
-          }
-
-          if (!acc[day]) {
-            return acc;
-          }
-
-          acc[day]![
-            latestStatus as
-              | "DELIVERED"
-              | "COMPLAINED"
-              | "OPENED"
-              | "CLICKED"
-              | "BOUNCED"
-          ]++;
-          return acc;
-        },
-        dateRecord
-      );
-
-      const emailDailyStatusCounts = Object.entries(_emailDailyStatusCounts)
-        .reverse()
-        .map(([date, counts]) => ({
-          name: date,
-          ...counts,
-        }));
-
-      return { emailStatusCounts, totalCount, emailDailyStatusCounts };
     }),
 
   getEmail: emailProcedure.query(async ({ input }) => {

apps/web/src/server/aws/ses.ts

@@ -162,15 +162,14 @@ export async function sendEmailThroughSes({
           ...(unsubUrl
             ? [
                 { Name: "List-Unsubscribe", Value: `<${unsubUrl}>` },
-                { Name: "List-Unsubscribe-Post", Value: "List-Unsubscribe=One-Click" },
+                {
+                  Name: "List-Unsubscribe-Post",
+                  Value: "List-Unsubscribe=One-Click",
+                },
               ]
-            : []
+            : []),
-          ),
           // Spread in the precedence header if present
-          ...(isBulk
+          ...(isBulk ? [{ Name: "Precedence", Value: "bulk" }] : []),
-            ? [{ Name: "Precedence", Value: "bulk" }]
-            : []
-          ),
         ],
       },
     },
@@ -216,7 +215,8 @@ export async function sendEmailWithAttachments({
   rawEmail += `To: ${Array.isArray(to) ? to.join(", ") : to}\n`;
   rawEmail += cc && cc.length ? `Cc: ${cc.join(", ")}\n` : "";
   rawEmail += bcc && bcc.length ? `Bcc: ${bcc.join(", ")}\n` : "";
-  rawEmail += replyTo && replyTo.length ? `Reply-To: ${replyTo.join(", ")}\n` : "";
+  rawEmail +=
+    replyTo && replyTo.length ? `Reply-To: ${replyTo.join(", ")}\n` : "";
   rawEmail += `Subject: ${subject}\n`;
   rawEmail += `MIME-Version: 1.0\n`;
   rawEmail += `Content-Type: multipart/mixed; boundary="${boundary}"\n\n`;
@@ -266,7 +266,7 @@ export async function addWebhookConfiguration(
   configName: string,
   topicArn: string,
   eventTypes: EventType[],
-  region: string,
+  region: string
 ) {
   const sesClient = getSesClient(region);
 

apps/web/src/server/service/ses-hook-parser.ts

@@ -3,6 +3,8 @@ import { SesClick, SesEvent, SesEventDataKey } from "~/types/aws-types";
 import { db } from "../db";
 import { updateCampaignAnalytics } from "./campaign-service";
 import { env } from "~/env";
+import { getRedis } from "../redis";
+import { Queue, Worker } from "bullmq";
 
 export async function parseSesHook(data: SesEvent) {
   const mailStatus = getEmailStatus(data);
@@ -45,6 +47,49 @@ export async function parseSesHook(data: SesEvent) {
       WHERE id = ${email.id}
     `;
 
+  // Update daily email usage statistics
+  const today = new Date().toISOString().split("T")[0] as string; // Format: YYYY-MM-DD
+
+  if (
+    [
+      "DELIVERED",
+      "OPENED",
+      "CLICKED",
+      "BOUNCED",
+      "COMPLAINED",
+      "SENT",
+    ].includes(mailStatus)
+  ) {
+    const updateField = mailStatus.toLowerCase();
+
+    await db.dailyEmailUsage.upsert({
+      where: {
+        teamId_domainId_date_type: {
+          teamId: email.teamId,
+          domainId: email.domainId ?? 0,
+          date: today,
+          type: email.campaignId ? "MARKETING" : "TRANSACTIONAL",
+        },
+      },
+      create: {
+        teamId: email.teamId,
+        domainId: email.domainId ?? 0,
+        date: today,
+        type: email.campaignId ? "MARKETING" : "TRANSACTIONAL",
+        delivered: updateField === "delivered" ? 1 : 0,
+        opened: updateField === "opened" ? 1 : 0,
+        clicked: updateField === "clicked" ? 1 : 0,
+        bounced: updateField === "bounced" ? 1 : 0,
+        complained: updateField === "complained" ? 1 : 0,
+      },
+      update: {
+        [updateField]: {
+          increment: 1,
+        },
+      },
+    });
+  }
+
   if (email.campaignId) {
     if (
       mailStatus !== "CLICKED" ||
@@ -109,3 +154,28 @@ function getEmailData(data: SesEvent) {
     return data[eventType.toLowerCase() as SesEventDataKey];
   }
 }
+
+export class SesHookParser {
+  private static sesHookQueue = new Queue("ses-web-hook", {
+    connection: getRedis(),
+  });
+
+  private static worker = new Worker(
+    "ses-web-hook",
+    async (job) => {
+      await this.execute(job.data);
+    },
+    {
+      connection: getRedis(),
+      concurrency: 200,
+    }
+  );
+
+  private static async execute(event: SesEvent) {
+    await parseSesHook(event);
+  }
+
+  static async queue(data: { event: SesEvent; messageId: string }) {
+    return await this.sesHookQueue.add(data.messageId, data.event);
+  }
+}