queue bulk contacts (#273)

apps/web/src/server/service/contact-queue-service.ts

@@ -0,0 +1,122 @@
+import { Queue, Worker } from "bullmq";
+import { getRedis } from "../redis";
+import {
+  DEFAULT_QUEUE_OPTIONS,
+  CONTACT_BULK_ADD_QUEUE,
+} from "../queue/queue-constants";
+import { logger } from "../logger/log";
+import { createWorkerHandler, TeamJob } from "../queue/bullmq-context";
+import { addOrUpdateContact, ContactInput } from "./contact-service";
+
+type ContactJobData = {
+  contactBookId: string;
+  contact: ContactInput;
+  teamId?: number;
+};
+
+type ContactJob = TeamJob<ContactJobData>;
+
+class ContactQueueService {
+  public static queue = new Queue<ContactJobData>(CONTACT_BULK_ADD_QUEUE, {
+    connection: getRedis(),
+    defaultJobOptions: DEFAULT_QUEUE_OPTIONS,
+  });
+
+  public static worker = new Worker(
+    CONTACT_BULK_ADD_QUEUE,
+    createWorkerHandler(processContactJob),
+    {
+      connection: getRedis(),
+      concurrency: 20,
+    },
+  );
+
+  static {
+    this.worker.on("error", (err) => {
+      logger.error({ err }, "[ContactQueueService]: Worker error");
+    });
+
+    logger.info("[ContactQueueService]: Initialized contact queue service");
+  }
+
+  public static async addContactJob(
+    contactBookId: string,
+    contact: ContactInput,
+    teamId?: number,
+    delay?: number,
+  ) {
+    await this.queue.add(
+      `add-contact-${contact.email}`,
+      {
+        contactBookId,
+        contact,
+        teamId,
+      },
+      {
+        delay,
+        ...DEFAULT_QUEUE_OPTIONS,
+      },
+    );
+  }
+
+  public static async addBulkContactJobs(
+    contactBookId: string,
+    contacts: ContactInput[],
+    teamId?: number,
+  ) {
+    const jobs = contacts.map((contact) => ({
+      name: `add-contact-${contact.email}`,
+      data: {
+        contactBookId,
+        contact,
+        teamId,
+      },
+      opts: DEFAULT_QUEUE_OPTIONS,
+    }));
+
+    await this.queue.addBulk(jobs);
+    logger.info(
+      { count: contacts.length, contactBookId },
+      "[ContactQueueService]: Added bulk contact jobs to queue",
+    );
+  }
+
+  public static async getQueueStats() {
+    const waiting = await this.queue.getWaiting();
+    const active = await this.queue.getActive();
+    const completed = await this.queue.getCompleted();
+    const failed = await this.queue.getFailed();
+
+    return {
+      waiting: waiting.length,
+      active: active.length,
+      completed: completed.length,
+      failed: failed.length,
+    };
+  }
+}
+
+async function processContactJob(job: ContactJob) {
+  const { contactBookId, contact } = job.data;
+
+  logger.info(
+    { contactEmail: contact.email, contactBookId },
+    "[ContactQueueService]: Processing contact job",
+  );
+
+  try {
+    await addOrUpdateContact(contactBookId, contact);
+    logger.info(
+      { contactEmail: contact.email },
+      "[ContactQueueService]: Successfully processed contact job",
+    );
+  } catch (error) {
+    logger.error(
+      { contactEmail: contact.email, error },
+      "[ContactQueueService]: Failed to process contact job",
+    );
+    throw error;
+  }
+}
+
+export { ContactQueueService };

apps/web/src/server/service/contact-service.ts

@@ -1,4 +1,5 @@
 import { db } from "../db";
+import { ContactQueueService } from "./contact-queue-service";
 
 export type ContactInput = {
   email: string;
@@ -60,13 +61,15 @@ export async function deleteContact(contactId: string) {
 
 export async function bulkAddContacts(
   contactBookId: string,
-  contacts: Array<ContactInput>
+  contacts: Array<ContactInput>,
+  teamId?: number
 ) {
-  const createdContacts = await Promise.all(
-    contacts.map((contact) => addOrUpdateContact(contactBookId, contact))
-  );
+  await ContactQueueService.addBulkContactJobs(contactBookId, contacts, teamId);
 
-  return createdContacts;
+  return {
+    message: `Queued ${contacts.length} contacts for processing`,
+    count: contacts.length,
+  };
 }
 
 export async function unsubscribeContact(contactId: string) {