add logging (#187)

apps/web/src/server/service/api-service.ts

@@ -3,6 +3,7 @@ import { db } from "../db";
 import { randomBytes } from "crypto";
 import { smallNanoid } from "../nanoid";
 import { createSecureHash, verifySecureHash } from "../crypto";
+import { logger } from "../logger/log";
 
 export async function addApiKey({
   name,
@@ -32,7 +33,7 @@ export async function addApiKey({
     });
     return apiKey;
   } catch (error) {
-    console.error("Error adding API key:", error);
+    logger.error({ err: error }, "Error adding API key");
     throw error;
   }
 }
@@ -64,7 +65,7 @@ export async function getTeamAndApiKey(apiKey: string) {
 
     return { team, apiKey: apiKeyRow };
   } catch (error) {
-    console.error("Error verifying API key:", error);
+    logger.error({ err: error }, "Error verifying API key");
     return null;
   }
 }
@@ -77,7 +78,7 @@ export async function deleteApiKey(id: number) {
       },
     });
   } catch (error) {
-    console.error("Error deleting API key:", error);
+    logger.error({ err: error }, "Error deleting API key");
     throw error;
   }
 }

apps/web/src/server/service/campaign-service.ts

@@ -16,6 +16,8 @@ import {
   CAMPAIGN_MAIL_PROCESSING_QUEUE,
   DEFAULT_QUEUE_OPTIONS,
 } from "../queue/queue-constants";
+import { logger } from "../logger/log";
+import { createWorkerHandler, TeamJob } from "../queue/bullmq-context";
 
 export async function sendCampaign(id: string) {
   let campaign = await db.campaign.findUnique({
@@ -41,7 +43,7 @@ export async function sendCampaign(id: string) {
       data: { html },
     });
   } catch (error) {
-    console.error(error);
+    logger.error({ err: error }, "Failed to parse campaign content");
     throw new Error("Failed to parse campaign content");
   }
 
@@ -158,7 +160,7 @@ export async function unsubscribeContact({
 
     return contact;
   } catch (error) {
-    console.error("Error unsubscribing contact:", error);
+    logger.error({ err: error }, "Error unsubscribing contact");
     throw new Error("Failed to unsubscribe contact");
   }
 }
@@ -207,7 +209,7 @@ export async function subscribeContact(id: string, hash: string) {
 
     return true;
   } catch (error) {
-    console.error("Error subscribing contact:", error);
+    logger.error({ err: error }, "Error subscribing contact");
     throw new Error("Failed to subscribe contact");
   }
 }
@@ -242,6 +244,8 @@ type CampaignEmailJob = {
   };
 };
 
+type QueueCampaignEmailJob = TeamJob<CampaignEmailJob>;
+
 async function processContactEmail(jobData: CampaignEmailJob) {
   const { contact, campaign, emailConfig } = jobData;
   const jsonContent = JSON.parse(campaign.content || "{}");
@@ -282,6 +286,7 @@ async function processContactEmail(jobData: CampaignEmailJob) {
   // Queue email for sending
   await EmailQueueService.queueEmail(
     email.id,
+    emailConfig.teamId,
     emailConfig.region,
     false,
     unsubscribeUrl
@@ -306,7 +311,7 @@ export async function sendCampaignEmail(
 
   const domain = await validateDomainFromEmail(from, teamId);
 
-  console.log("Bulk queueing contacts");
+  logger.info("Bulk queueing contacts");
 
   await CampaignEmailService.queueBulkContacts(
     contacts.map((contact) => ({
@@ -382,15 +387,19 @@ export async function updateCampaignAnalytics(
 const CAMPAIGN_EMAIL_CONCURRENCY = 50;
 
 class CampaignEmailService {
-  private static campaignQueue = new Queue(CAMPAIGN_MAIL_PROCESSING_QUEUE, {
-    connection: getRedis(),
-  });
+  private static campaignQueue = new Queue<QueueCampaignEmailJob>(
+    CAMPAIGN_MAIL_PROCESSING_QUEUE,
+    {
+      connection: getRedis(),
+    }
+  );
 
+  // TODO: Add team context to job data when queueing
   static worker = new Worker(
     CAMPAIGN_MAIL_PROCESSING_QUEUE,
-    async (job) => {
+    createWorkerHandler(async (job: QueueCampaignEmailJob) => {
       await processContactEmail(job.data);
-    },
+    }),
     {
       connection: getRedis(),
       concurrency: CAMPAIGN_EMAIL_CONCURRENCY,
@@ -400,7 +409,10 @@ class CampaignEmailService {
   static async queueContact(data: CampaignEmailJob) {
     return await this.campaignQueue.add(
       `contact-${data.contact.id}`,
-      data,
+      {
+        ...data,
+        teamId: data.emailConfig.teamId,
+      },
       DEFAULT_QUEUE_OPTIONS
     );
   }
@@ -409,7 +421,10 @@ class CampaignEmailService {
     return await this.campaignQueue.addBulk(
       data.map((item) => ({
         name: `contact-${item.contact.id}`,
-        data: item,
+        data: {
+          ...item,
+          teamId: item.emailConfig.teamId,
+        },
         opts: {
           ...DEFAULT_QUEUE_OPTIONS,
         },

apps/web/src/server/service/domain-service.ts

@@ -5,6 +5,7 @@ import * as ses from "~/server/aws/ses";
 import { db } from "~/server/db";
 import { SesSettingsService } from "./ses-settings-service";
 import { UnsendApiError } from "../public-api/api-error";
+import { logger } from "../logger/log";
 
 const dnsResolveTxt = util.promisify(dns.resolveTxt);
 
@@ -60,7 +61,7 @@ export async function createDomain(
 ) {
   const domainStr = tldts.getDomain(name);
 
-  console.log("Creating domain", { domainStr, name, region });
+  logger.info({ domainStr, name, region }, "Creating domain");
 
   if (!domainStr) {
     throw new Error("Invalid domain");
@@ -191,7 +192,7 @@ async function getDmarcRecord(domain: string) {
     const dmarcRecord = await dnsResolveTxt(`_dmarc.${domain}`);
     return dmarcRecord;
   } catch (error) {
-    console.error("Error fetching DMARC record:", error);
+    logger.error({ err: error, domain }, "Error fetching DMARC record");
     return null; // or handle error as appropriate
   }
 }

apps/web/src/server/service/email-queue-service.ts

@@ -8,6 +8,15 @@ import { sendRawEmail } from "../aws/ses";
 import { getRedis } from "../redis";
 import { DEFAULT_QUEUE_OPTIONS } from "../queue/queue-constants";
 import { Prisma } from "@prisma/client";
+import { logger } from "../logger/log";
+import { createWorkerHandler, TeamJob } from "../queue/bullmq-context";
+
+type QueueEmailJob = TeamJob<{
+  emailId: string;
+  timestamp: number;
+  unsubUrl?: string;
+  isBulk?: boolean;
+}>;
 
 function createQueueAndWorker(region: string, quota: number, suffix: string) {
   const connection = getRedis();
@@ -16,7 +25,8 @@ function createQueueAndWorker(region: string, quota: number, suffix: string) {
 
   const queue = new Queue(queueName, { connection });
 
-  const worker = new Worker(queueName, executeEmail, {
+  // TODO: Add team context to job data when queueing
+  const worker = new Worker(queueName, createWorkerHandler(executeEmail), {
     concurrency: quota,
     connection,
   });
@@ -26,9 +36,9 @@ function createQueueAndWorker(region: string, quota: number, suffix: string) {
 
 export class EmailQueueService {
   private static initialized = false;
-  public static transactionalQueue = new Map<string, Queue>();
+  public static transactionalQueue = new Map<string, Queue<QueueEmailJob>>();
   private static transactionalWorker = new Map<string, Worker>();
-  public static marketingQueue = new Map<string, Queue>();
+  public static marketingQueue = new Map<string, Queue<QueueEmailJob>>();
   private static marketingWorker = new Map<string, Worker>();
 
   public static initializeQueue(
@@ -36,7 +46,10 @@ export class EmailQueueService {
     quota: number,
     transactionalQuotaPercentage: number
   ) {
-    console.log(`[EmailQueueService]: Initializing queue for region ${region}`);
+    logger.info(
+      { region },
+      `[EmailQueueService]: Initializing queue for region`
+    );
 
     const transactionalQuota = Math.floor(
       (quota * transactionalQuotaPercentage) / 100
@@ -44,8 +57,9 @@ export class EmailQueueService {
     const marketingQuota = quota - transactionalQuota;
 
     if (this.transactionalQueue.has(region)) {
-      console.log(
-        `[EmailQueueService]: Updating transactional quota for region ${region} to ${transactionalQuota}`
+      logger.info(
+        { region, transactionalQuota },
+        `[EmailQueueService]: Updating transactional quota for region`
       );
       const transactionalWorker = this.transactionalWorker.get(region);
       if (transactionalWorker) {
@@ -53,8 +67,9 @@ export class EmailQueueService {
           transactionalQuota !== 0 ? transactionalQuota : 1;
       }
     } else {
-      console.log(
-        `[EmailQueueService]: Creating transactional queue for region ${region} with quota ${transactionalQuota}`
+      logger.info(
+        { region, transactionalQuota },
+        `[EmailQueueService]: Creating transactional queue for region`
       );
       const { queue: transactionalQueue, worker: transactionalWorker } =
         createQueueAndWorker(
@@ -67,16 +82,18 @@ export class EmailQueueService {
     }
 
     if (this.marketingQueue.has(region)) {
-      console.log(
-        `[EmailQueueService]: Updating marketing quota for region ${region} to ${marketingQuota}`
+      logger.info(
+        { region, marketingQuota },
+        `[EmailQueueService]: Updating marketing quota for region`
       );
       const marketingWorker = this.marketingWorker.get(region);
       if (marketingWorker) {
         marketingWorker.concurrency = marketingQuota !== 0 ? marketingQuota : 1;
       }
     } else {
-      console.log(
-        `[EmailQueueService]: Creating marketing queue for region ${region} with quota ${marketingQuota}`
+      logger.info(
+        { region, marketingQuota },
+        `[EmailQueueService]: Creating marketing queue for region`
       );
       const { queue: marketingQueue, worker: marketingWorker } =
         createQueueAndWorker(
@@ -91,6 +108,7 @@ export class EmailQueueService {
 
   public static async queueEmail(
     emailId: string,
+    teamId: number,
     region: string,
     transactional: boolean,
     unsubUrl?: string,
@@ -108,7 +126,7 @@ export class EmailQueueService {
     }
     queue.add(
       emailId,
-      { emailId, timestamp: Date.now(), unsubUrl, isBulk },
+      { emailId, timestamp: Date.now(), unsubUrl, isBulk, teamId },
       { jobId: emailId, delay, ...DEFAULT_QUEUE_OPTIONS }
     );
   }
@@ -123,6 +141,7 @@ export class EmailQueueService {
   public static async queueBulk(
     jobs: {
       emailId: string;
+      teamId: number;
       region: string;
       transactional: boolean;
       unsubUrl?: string;
@@ -131,7 +150,7 @@ export class EmailQueueService {
     }[]
   ): Promise<void> {
     if (jobs.length === 0) {
-      console.log("[EmailQueueService]: No jobs provided for bulk queue.");
+      logger.info("[EmailQueueService]: No jobs provided for bulk queue.");
       return;
     }
 
@@ -139,8 +158,9 @@ export class EmailQueueService {
       await this.init();
     }
 
-    console.log(
-      `[EmailQueueService]: Starting bulk queue for ${jobs.length} jobs.`
+    logger.info(
+      { count: jobs.length },
+      `[EmailQueueService]: Starting bulk queue for jobs.`
     );
 
     // Group jobs by region and type
@@ -176,8 +196,9 @@ export class EmailQueueService {
     for (const groupKey in groupedJobs) {
       const group = groupedJobs[groupKey];
       if (!group || !group.queue) {
-        console.error(
-          `[EmailQueueService]: Queue not found for group ${groupKey} during bulk add. Skipping ${group?.jobDetails?.length ?? 0} jobs.`
+        logger.error(
+          { groupKey, count: group?.jobDetails?.length ?? 0 },
+          `[EmailQueueService]: Queue not found for group during bulk add. Skipping jobs.`
         );
         // Optionally: handle these skipped jobs (e.g., mark corresponding emails as failed)
         continue;
@@ -192,6 +213,7 @@ export class EmailQueueService {
           timestamp: job.timestamp ?? Date.now(),
           unsubUrl: job.unsubUrl,
           isBulk,
+          teamId: job.teamId,
         },
         opts: {
           jobId: job.emailId, // Use emailId as jobId
@@ -200,14 +222,15 @@ export class EmailQueueService {
         },
       }));
 
-      console.log(
-        `[EmailQueueService]: Adding ${bulkData.length} jobs to queue ${queue.name}`
+      logger.info(
+        { count: bulkData.length, queue: queue.name },
+        `[EmailQueueService]: Adding jobs to queue`
       );
       bulkAddPromises.push(
         queue.addBulk(bulkData).catch((error) => {
-          console.error(
-            `[EmailQueueService]: Failed to add bulk jobs to queue ${queue.name}:`,
-            error
+          logger.error(
+            { err: error, queue: queue.name },
+            `[EmailQueueService]: Failed to add bulk jobs to queue`
           );
           // Optionally: handle bulk add failure (e.g., mark corresponding emails as failed)
         })
@@ -215,7 +238,7 @@ export class EmailQueueService {
     }
 
     await Promise.allSettled(bulkAddPromises);
-    console.log(
+    logger.info(
       "[EmailQueueService]: Finished processing bulk queue requests."
     );
   }
@@ -278,16 +301,10 @@ export class EmailQueueService {
   }
 }
 
-async function executeEmail(
-  job: Job<{
-    emailId: string;
-    timestamp: number;
-    unsubUrl?: string;
-    isBulk?: boolean;
-  }>
-) {
-  console.log(
-    `[EmailQueueService]: Executing email job ${job.data.emailId}, time elapsed: ${Date.now() - job.data.timestamp}ms`
+async function executeEmail(job: QueueEmailJob) {
+  logger.info(
+    { emailId: job.data.emailId, elapsed: Date.now() - job.data.timestamp },
+    `[EmailQueueService]: Executing email job`
   );
 
   const email = await db.email.findUnique({
@@ -301,7 +318,10 @@ async function executeEmail(
     : null;
 
   if (!email) {
-    console.log(`[EmailQueueService]: Email not found, skipping`);
+    logger.info(
+      { emailId: job.data.emailId },
+      `[EmailQueueService]: Email not found, skipping`
+    );
     return;
   }
 
@@ -309,7 +329,7 @@ async function executeEmail(
     ? JSON.parse(email.attachments)
     : [];
 
-  console.log(`Domain: ${JSON.stringify(domain)}`);
+  logger.info({ domain }, `Domain`);
 
   const configurationSetName = await getConfigurationSetName(
     domain?.clickTracking ?? false,
@@ -321,7 +341,7 @@ async function executeEmail(
     return;
   }
 
-  console.log(`[EmailQueueService]: Sending email ${email.id}`);
+  logger.info({ emailId: email.id }, `[EmailQueueService]: Sending email`);
   const unsubUrl = job.data.unsubUrl;
   const isBulk = job.data.isBulk;
 

apps/web/src/server/service/email-service.ts

@@ -4,6 +4,7 @@ import { UnsendApiError } from "~/server/public-api/api-error";
 import { EmailQueueService } from "./email-queue-service";
 import { validateDomainFromEmail } from "./domain-service";
 import { EmailRenderer } from "@unsend/email-editor/src/renderer";
+import { logger } from "../logger/log";
 
 async function checkIfValidEmail(emailId: string) {
   const email = await db.email.findUnique({
@@ -155,6 +156,7 @@ export async function sendEmail(
   try {
     await EmailQueueService.queueEmail(
       email.id,
+      teamId,
       domain.region,
       true,
       undefined,
@@ -399,15 +401,16 @@ export async function sendBulkEmails(
         // Prepare queue job
         queueJobs.push({
           emailId: email.id,
+          teamId,
           region: domain.region,
           transactional: true, // Bulk emails are still transactional
           delay,
           timestamp: Date.now(),
         });
       } catch (error: any) {
-        console.error(
-          `Failed to create email record for recipient ${to}:`,
-          error
+        logger.error(
+          { err: error, to },
+          `Failed to create email record for recipient`
         );
         // Continue processing other emails
       }

apps/web/src/server/service/notification-service.ts

@@ -1,11 +1,12 @@
 import { env } from "~/env";
+import { logger } from "../logger/log";
 
 export async function sendToDiscord(message: string) {
   if (!env.DISCORD_WEBHOOK_URL) {
-    console.error(
+    logger.error(
       "Discord webhook URL is not defined in the environment variables. So printing the message to the console."
     );
-    console.log("Message: ", message);
+    logger.info({ message }, "Message");
     return;
   }
 
@@ -19,9 +20,12 @@ export async function sendToDiscord(message: string) {
   });
 
   if (response.ok) {
-    console.log("Message sent to Discord successfully.");
+    logger.info("Message sent to Discord successfully.");
   } else {
-    console.error("Failed to send message to Discord:", response.statusText);
+    logger.error(
+      { statusText: response.statusText },
+      "Failed to send message to Discord:"
+    );
   }
 
   return;

apps/web/src/server/service/ses-hook-parser.ts

@@ -17,12 +17,14 @@ import {
   DEFAULT_QUEUE_OPTIONS,
   SES_WEBHOOK_QUEUE,
 } from "../queue/queue-constants";
+import { getChildLogger, logger, withLogger } from "../logger/log";
+import { randomUUID } from "crypto";
 
 export async function parseSesHook(data: SesEvent) {
   const mailStatus = getEmailStatus(data);
 
   if (!mailStatus) {
-    console.error("Unknown email status", data);
+    logger.error({ data }, "Unknown email status");
     return false;
   }
 
@@ -36,8 +38,14 @@ export async function parseSesHook(data: SesEvent) {
     },
   });
 
+  logger.setBindings({
+    sesEmailId,
+    mailId: email?.id,
+    teamId: email?.teamId,
+  });
+
   if (!email) {
-    console.error("Email not found", data);
+    logger.error({ data }, "Email not found");
     return false;
   }
 
@@ -286,7 +294,14 @@ export class SesHookParser {
   private static worker = new Worker(
     SES_WEBHOOK_QUEUE,
     async (job) => {
-      await this.execute(job.data);
+      return await withLogger(
+        getChildLogger({
+          queueId: job.id ?? randomUUID(),
+        }),
+        async () => {
+          await this.execute(job.data);
+        }
+      );
     },
     {
       connection: getRedis(),

apps/web/src/server/service/ses-settings-service.ts

@@ -6,6 +6,7 @@ import * as ses from "~/server/aws/ses";
 import { EventType } from "@aws-sdk/client-sesv2";
 import { EmailQueueService } from "./email-queue-service";
 import { smallNanoid } from "../nanoid";
+import { logger } from "../logger/log";
 
 const GENERAL_EVENTS: EventType[] = [
   "BOUNCE",
@@ -121,9 +122,12 @@ export class SesSettingsService {
         setting.sesEmailRateLimit,
         setting.transactionalQuota
       );
-      console.log(
-        EmailQueueService.transactionalQueue,
-        EmailQueueService.marketingQueue
+      logger.info(
+        {
+          transactionalQueue: EmailQueueService.transactionalQueue,
+          marketingQueue: EmailQueueService.marketingQueue,
+        },
+        "Email queues initialized"
       );
 
       await this.invalidateCache();
@@ -132,10 +136,13 @@ export class SesSettingsService {
         try {
           await sns.deleteTopic(topicArn, region);
         } catch (deleteError) {
-          console.error("Failed to delete SNS topic after error:", deleteError);
+          logger.error(
+            { err: deleteError },
+            "Failed to delete SNS topic after error"
+          );
         }
       }
-      console.error("Failed to create SES setting", error);
+      logger.error({ err: error }, "Failed to create SES setting");
       throw error;
     }
   }
@@ -160,9 +167,12 @@ export class SesSettingsService {
         sesEmailRateLimit: sendingRateLimit,
       },
     });
-    console.log(
-      EmailQueueService.transactionalQueue,
-      EmailQueueService.marketingQueue
+    logger.info(
+      {
+        transactionalQueue: EmailQueueService.transactionalQueue,
+        marketingQueue: EmailQueueService.marketingQueue,
+      },
+      "Email queues before update"
     );
 
     EmailQueueService.initializeQueue(
@@ -171,9 +181,12 @@ export class SesSettingsService {
       setting.transactionalQuota
     );
 
-    console.log(
-      EmailQueueService.transactionalQueue,
-      EmailQueueService.marketingQueue
+    logger.info(
+      {
+        transactionalQueue: EmailQueueService.transactionalQueue,
+        marketingQueue: EmailQueueService.marketingQueue,
+      },
+      "Email queues after update"
     );
 
     await this.invalidateCache();
@@ -261,7 +274,7 @@ async function registerConfigurationSet(setting: SesSetting) {
 }
 
 async function isValidUnsendUrl(url: string) {
-  console.log("Checking if URL is valid", url);
+  logger.info({ url }, "Checking if URL is valid");
   try {
     const response = await fetch(`${url}/api/ses_callback`, {
       method: "GET",
@@ -272,7 +285,7 @@ async function isValidUnsendUrl(url: string) {
       error: response.statusText,
     };
   } catch (e) {
-    console.log("Error checking if URL is valid", e);
+    logger.error({ err: e }, "Error checking if URL is valid");
     return {
       isValid: false,
       code: 500,