diff --git a/apps/docs/guides/webhooks.mdx b/apps/docs/guides/webhooks.mdx index 36ec7b4..eb5a521 100644 --- a/apps/docs/guides/webhooks.mdx +++ b/apps/docs/guides/webhooks.mdx @@ -35,20 +35,21 @@ Webhooks allow you to receive HTTP POST requests to your server when events occu ### Email events -| Event | Description | -| ------------------------ | ---------------------------------------------------- | -| `email.queued` | Email has been queued for sending | -| `email.sent` | Email has been sent to the recipient's mail server | -| `email.delivered` | Email was successfully delivered | -| `email.delivery_delayed` | Email delivery is being retried | -| `email.bounced` | Email bounced (permanent or temporary) | -| `email.rejected` | Email was rejected | -| `email.complained` | Recipient marked email as spam | -| `email.failed` | Email failed to send | -| `email.cancelled` | Scheduled email was cancelled | -| `email.suppressed` | Email was suppressed (recipient on suppression list) | -| `email.opened` | Recipient opened the email | -| `email.clicked` | Recipient clicked a link in the email | +| Event | Description | +| ------------------------- | ---------------------------------------------------- | +| `email.queued` | Email has been queued for sending | +| `email.sent` | Email has been sent to the recipient's mail server | +| `email.delivered` | Email was successfully delivered | +| `email.delivery_delayed` | Email delivery is being retried | +| `email.bounced` | Email bounced (permanent or temporary) | +| `email.rejected` | Email was rejected | +| `email.rendering_failure` | Email failed during template rendering | +| `email.complained` | Recipient marked email as spam | +| `email.failed` | Email failed to send | +| `email.cancelled` | Scheduled email was cancelled | +| `email.suppressed` | Email was suppressed (recipient on suppression list) | +| `email.opened` | Recipient opened the email | +| `email.clicked` | Recipient clicked a link in the email | ### Contact events @@ -276,9 +277,9 @@ If your endpoint doesn't return a 2xx response, useSend will retry delivery with After 6 failed attempts, the webhook call is marked as failed. - If your webhook endpoint fails 30 consecutive times across any calls, the - webhook will be automatically disabled to prevent continued failures. You can - re-enable it from the dashboard. + If your webhook endpoint fails 30 consecutive calls, the webhook will be + automatically disabled to prevent continued failures. You can re-enable it + from the dashboard. ## Best practices @@ -339,9 +340,9 @@ The test event will have type `webhook.test` with the following payload: "${timestamp}.${rawBody}")` - After 30 consecutive failures, webhooks are automatically disabled. Fix the - issue with your endpoint, then re-enable the webhook from the dashboard. The - failure counter resets on the next successful delivery. + After 30 consecutive failed calls, webhooks are automatically disabled. Fix + the issue with your endpoint, then re-enable the webhook from the dashboard. + The failure counter resets on the next successful delivery. diff --git a/apps/web/src/app/(dashboard)/webhooks/[webhookId]/page.tsx b/apps/web/src/app/(dashboard)/webhooks/[webhookId]/page.tsx index 4d588dd..9ef03fc 100644 --- a/apps/web/src/app/(dashboard)/webhooks/[webhookId]/page.tsx +++ b/apps/web/src/app/(dashboard)/webhooks/[webhookId]/page.tsx @@ -56,6 +56,7 @@ function WebhookDetailActions({ const [open, setOpen] = useState(false); const isPaused = webhook.status === "PAUSED"; const isAutoDisabled = webhook.status === "AUTO_DISABLED"; + const canActivate = isPaused || isAutoDisabled; return ( @@ -100,12 +101,12 @@ function WebhookDetailActions({ onToggleStatus(); setOpen(false); }} - disabled={isToggling || isAutoDisabled} + disabled={isToggling} > - {isPaused ? ( + {canActivate ? ( <> - Resume + {isAutoDisabled ? "Re-enable" : "Resume"} ) : ( <> diff --git a/apps/web/src/app/(dashboard)/webhooks/webhook-list.tsx b/apps/web/src/app/(dashboard)/webhooks/webhook-list.tsx index 11c6ab2..b42f607 100644 --- a/apps/web/src/app/(dashboard)/webhooks/webhook-list.tsx +++ b/apps/web/src/app/(dashboard)/webhooks/webhook-list.tsx @@ -160,6 +160,7 @@ function WebhookActions({ const [open, setOpen] = useState(false); const isPaused = webhook.status === "PAUSED"; const isAutoDisabled = webhook.status === "AUTO_DISABLED"; + const canActivate = isPaused || isAutoDisabled; return ( @@ -190,12 +191,12 @@ function WebhookActions({ onToggleStatus(); setOpen(false); }} - disabled={isToggling || isAutoDisabled} + disabled={isToggling} > - {isPaused ? ( + {canActivate ? ( <> - Resume + {isAutoDisabled ? "Re-enable" : "Resume"} ) : ( <> diff --git a/apps/web/src/server/service/webhook-service.ts b/apps/web/src/server/service/webhook-service.ts index 3888c7f..9b32704 100644 --- a/apps/web/src/server/service/webhook-service.ts +++ b/apps/web/src/server/service/webhook-service.ts @@ -513,18 +513,38 @@ async function processWebhookCall(job: WebhookCallJob) { ? new Date(Date.now() + computeBackoff(attempt)) : null; - const updatedWebhook = await db.webhook.update({ - where: { id: call.webhookId }, - data: { - consecutiveFailures: { - increment: 1, + const isFinalAttempt = attempt >= WEBHOOK_MAX_ATTEMPTS; + + const updatedWebhook = await db.$transaction(async (tx) => { + const webhookAfterFailure = await tx.webhook.update({ + where: { id: call.webhookId }, + data: { + lastFailureAt: new Date(), + ...(isFinalAttempt + ? { + consecutiveFailures: { + increment: 1, + }, + } + : {}), }, - lastFailureAt: new Date(), - status: - call.webhook.consecutiveFailures + 1 >= WEBHOOK_AUTO_DISABLE_THRESHOLD - ? WebhookStatus.AUTO_DISABLED - : call.webhook.status, - }, + }); + + if ( + isFinalAttempt && + webhookAfterFailure.status === WebhookStatus.ACTIVE && + webhookAfterFailure.consecutiveFailures >= + WEBHOOK_AUTO_DISABLE_THRESHOLD + ) { + return tx.webhook.update({ + where: { id: call.webhookId }, + data: { + status: WebhookStatus.AUTO_DISABLED, + }, + }); + } + + return webhookAfterFailure; }); await db.webhookCall.update({ diff --git a/apps/web/src/server/service/webhook-service.unit.test.ts b/apps/web/src/server/service/webhook-service.unit.test.ts new file mode 100644 index 0000000..15cefba --- /dev/null +++ b/apps/web/src/server/service/webhook-service.unit.test.ts @@ -0,0 +1,377 @@ +import { WebhookCallStatus, WebhookStatus } from "@prisma/client"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { + capturedProcessWebhookCall, + mockDb, + mockLogger, + mockQueueAdd, + mockRedis, + mockTxWebhookUpdate, +} = vi.hoisted(() => ({ + capturedProcessWebhookCall: { + handler: null as any, + }, + mockDb: { + $transaction: vi.fn(), + webhook: { + create: vi.fn(), + delete: vi.fn(), + findFirst: vi.fn(), + findMany: vi.fn(), + update: vi.fn(), + }, + webhookCall: { + create: vi.fn(), + findFirst: vi.fn(), + findMany: vi.fn(), + findUnique: vi.fn(), + update: vi.fn(), + }, + }, + mockLogger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }, + mockQueueAdd: vi.fn(), + mockRedis: { + eval: vi.fn(), + set: vi.fn(), + }, + mockTxWebhookUpdate: vi.fn(), +})); + +vi.mock("bullmq", () => ({ + Queue: class { + public add = mockQueueAdd; + }, + Worker: class { + public on = vi.fn(); + }, +})); + +vi.mock("~/server/db", () => ({ + db: mockDb, +})); + +vi.mock("~/server/logger/log", () => ({ + logger: mockLogger, +})); + +vi.mock("~/server/service/limit-service", () => ({ + LimitService: { + checkWebhookLimit: vi.fn(), + }, +})); + +vi.mock("~/server/queue/bullmq-context", () => ({ + createWorkerHandler: (handler: any) => { + capturedProcessWebhookCall.handler = handler; + return handler; + }, +})); + +vi.mock("~/server/redis", () => ({ + getRedis: () => mockRedis, +})); + +import { WebhookService } from "~/server/service/webhook-service"; + +function buildCall(overrides?: { + consecutiveFailures?: number; + status?: WebhookStatus; +}) { + return { + id: "call_123", + webhookId: "wh_123", + teamId: 77, + type: "email.delivered", + payload: JSON.stringify({ id: "email_123" }), + createdAt: new Date("2026-01-01T00:00:00.000Z"), + webhook: { + id: "wh_123", + url: "https://example.com/webhook", + secret: "whsec_test", + apiVersion: null, + status: overrides?.status ?? WebhookStatus.ACTIVE, + consecutiveFailures: overrides?.consecutiveFailures ?? 0, + }, + }; +} + +async function invokeProcessWebhookCall(attemptsMade = 0) { + if (!capturedProcessWebhookCall.handler) { + throw new Error("processWebhookCall handler not captured"); + } + + return capturedProcessWebhookCall.handler({ + attemptsMade, + data: { + callId: "call_123", + teamId: 77, + }, + }); +} + +describe("WebhookService documented behavior", () => { + beforeEach(() => { + mockDb.webhook.create.mockReset(); + mockDb.webhook.delete.mockReset(); + mockDb.webhook.findFirst.mockReset(); + mockDb.webhook.findMany.mockReset(); + mockDb.webhook.update.mockReset(); + + mockDb.webhookCall.create.mockReset(); + mockDb.webhookCall.findFirst.mockReset(); + mockDb.webhookCall.findMany.mockReset(); + mockDb.webhookCall.findUnique.mockReset(); + mockDb.webhookCall.update.mockReset(); + + mockDb.$transaction.mockReset(); + mockLogger.debug.mockReset(); + mockLogger.error.mockReset(); + mockLogger.info.mockReset(); + mockLogger.warn.mockReset(); + mockQueueAdd.mockReset(); + mockRedis.eval.mockReset(); + mockRedis.set.mockReset(); + mockTxWebhookUpdate.mockReset(); + + mockRedis.set.mockResolvedValue("OK"); + mockRedis.eval.mockResolvedValue(1); + mockQueueAdd.mockResolvedValue(undefined); + mockDb.webhookCall.update.mockResolvedValue({}); + mockDb.webhook.update.mockResolvedValue({ + id: "wh_123", + status: WebhookStatus.ACTIVE, + consecutiveFailures: 0, + }); + + mockDb.$transaction.mockImplementation(async (input: unknown) => { + if (typeof input === "function") { + return input({ + webhook: { + update: mockTxWebhookUpdate, + }, + }); + } + + return Promise.all(input as Array>); + }); + }); + + it("sends documented webhook headers with retry=false on first attempt", async () => { + mockDb.webhookCall.findUnique.mockResolvedValue(buildCall()); + + const fetchSpy = vi.spyOn(global, "fetch").mockResolvedValue( + new Response('{"ok":true}', { + status: 200, + headers: { + "content-type": "application/json", + }, + }), + ); + + await expect(invokeProcessWebhookCall(0)).resolves.toBeUndefined(); + + const [, request] = fetchSpy.mock.calls[0]!; + const headers = request!.headers as Record; + expect(headers["X-UseSend-Event"]).toBe("email.delivered"); + expect(headers["X-UseSend-Call"]).toBe("call_123"); + expect(headers["X-UseSend-Signature"]).toMatch(/^v1=/); + expect(headers["X-UseSend-Timestamp"]).toBeTypeOf("string"); + expect(headers["X-UseSend-Retry"]).toBe("false"); + }); + + it("sets retry=true header for retry attempts", async () => { + mockDb.webhookCall.findUnique.mockResolvedValue(buildCall()); + + const fetchSpy = vi.spyOn(global, "fetch").mockResolvedValue( + new Response("ok", { + status: 200, + headers: { + "content-type": "text/plain", + }, + }), + ); + + await expect(invokeProcessWebhookCall(1)).resolves.toBeUndefined(); + + const [, request] = fetchSpy.mock.calls[0]!; + const headers = request!.headers as Record; + expect(headers["X-UseSend-Retry"]).toBe("true"); + }); + + it("marks webhook call as FAILED after 6 attempts", async () => { + mockDb.webhookCall.findUnique.mockResolvedValue(buildCall()); + mockTxWebhookUpdate.mockResolvedValue({ + id: "wh_123", + status: WebhookStatus.ACTIVE, + consecutiveFailures: 1, + }); + + vi.spyOn(global, "fetch").mockRejectedValue(new Error("network down")); + + await expect(invokeProcessWebhookCall(5)).rejects.toThrow("network down"); + + expect(mockDb.webhookCall.update).toHaveBeenLastCalledWith({ + where: { id: "call_123" }, + data: expect.objectContaining({ + status: WebhookCallStatus.FAILED, + attempt: 6, + nextAttemptAt: null, + }), + }); + }); + + it("does not increment consecutive failure counter before final attempt", async () => { + mockDb.webhookCall.findUnique.mockResolvedValue(buildCall()); + mockTxWebhookUpdate.mockResolvedValue({ + id: "wh_123", + status: WebhookStatus.ACTIVE, + consecutiveFailures: 0, + }); + + vi.spyOn(global, "fetch").mockRejectedValue(new Error("network down")); + + await expect(invokeProcessWebhookCall(0)).rejects.toThrow("network down"); + + expect(mockTxWebhookUpdate).toHaveBeenCalledTimes(1); + const firstUpdateInput = mockTxWebhookUpdate.mock.calls[0]![0] as { + data: { consecutiveFailures?: { increment: number } }; + }; + expect(firstUpdateInput.data.consecutiveFailures).toBeUndefined(); + expect(mockDb.webhookCall.update).toHaveBeenLastCalledWith({ + where: { id: "call_123" }, + data: expect.objectContaining({ + status: WebhookCallStatus.PENDING, + attempt: 1, + }), + }); + }); + + it("auto-disables only when the persisted failure count reaches 30", async () => { + mockDb.webhookCall.findUnique.mockResolvedValue( + buildCall({ consecutiveFailures: 29 }), + ); + mockTxWebhookUpdate + .mockResolvedValueOnce({ + id: "wh_123", + status: WebhookStatus.ACTIVE, + consecutiveFailures: 30, + }) + .mockResolvedValueOnce({ + id: "wh_123", + status: WebhookStatus.AUTO_DISABLED, + consecutiveFailures: 30, + }); + + vi.spyOn(global, "fetch").mockRejectedValue(new Error("endpoint 500")); + + await expect(invokeProcessWebhookCall(5)).resolves.toBeUndefined(); + + expect(mockTxWebhookUpdate).toHaveBeenCalledTimes(2); + expect(mockTxWebhookUpdate).toHaveBeenLastCalledWith({ + where: { id: "wh_123" }, + data: { + status: WebhookStatus.AUTO_DISABLED, + }, + }); + }); + + it("uses the latest persisted failure count when deciding auto-disable", async () => { + mockDb.webhookCall.findUnique.mockResolvedValue( + buildCall({ consecutiveFailures: 29 }), + ); + mockTxWebhookUpdate.mockResolvedValueOnce({ + id: "wh_123", + status: WebhookStatus.ACTIVE, + consecutiveFailures: 1, + }); + + vi.spyOn(global, "fetch").mockRejectedValue(new Error("endpoint 500")); + + await expect(invokeProcessWebhookCall(5)).rejects.toThrow("endpoint 500"); + expect(mockTxWebhookUpdate).toHaveBeenCalledTimes(1); + }); + + it("resets failure counter when re-enabling webhook", async () => { + mockDb.webhook.findFirst.mockResolvedValue({ + id: "wh_123", + teamId: 77, + consecutiveFailures: 12, + status: WebhookStatus.AUTO_DISABLED, + }); + mockDb.webhook.update.mockResolvedValue({ + id: "wh_123", + status: WebhookStatus.ACTIVE, + consecutiveFailures: 0, + }); + + await WebhookService.setWebhookStatus({ + id: "wh_123", + teamId: 77, + status: WebhookStatus.ACTIVE, + }); + + expect(mockDb.webhook.update).toHaveBeenCalledWith({ + where: { id: "wh_123" }, + data: { + status: WebhookStatus.ACTIVE, + consecutiveFailures: 0, + }, + }); + }); + + it("creates webhook.test payload from dashboard test trigger", async () => { + mockDb.webhook.findFirst.mockResolvedValue({ + id: "wh_123", + teamId: 77, + }); + mockDb.webhookCall.create.mockResolvedValue({ + id: "call_test_1", + }); + + await expect( + WebhookService.testWebhook({ + webhookId: "wh_123", + teamId: 77, + }), + ).resolves.toBe("call_test_1"); + + expect(mockDb.webhookCall.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + webhookId: "wh_123", + teamId: 77, + type: "webhook.test", + status: WebhookCallStatus.PENDING, + attempt: 0, + }), + }); + + const createInput = mockDb.webhookCall.create.mock.calls[0]![0] as { + data: { payload: string }; + }; + const payload = JSON.parse(createInput.data.payload) as { + sentAt: string; + test: boolean; + webhookId: string; + }; + + expect(payload).toMatchObject({ + test: true, + webhookId: "wh_123", + }); + expect(payload.sentAt).toBeTypeOf("string"); + expect(mockQueueAdd).toHaveBeenCalledWith( + "call_test_1", + { + callId: "call_test_1", + teamId: 77, + }, + { jobId: "call_test_1" }, + ); + }); +});