add postmortem (#241)

This commit is contained in:
KM Koushik
2025-09-19 22:09:54 +10:00
committed by GitHub
parent 2fa8c1b600
commit 1226e89aaf
6 changed files with 300 additions and 954 deletions

View File

@@ -1,3 +1,5 @@
import createMDX from "@next/mdx";
/** @type {import("next").NextConfig} */
const config = {
// Use static export in production by default; keep dev server dynamic
@@ -6,6 +8,11 @@ const config = {
// Required for static export if using images
unoptimized: true,
},
pageExtensions: ["js", "jsx", "md", "mdx", "ts", "tsx"],
};
export default config;
const withMDX = createMDX({
extension: /\.(md|mdx)$/,
});
export default withMDX(config);

View File

@@ -10,6 +10,10 @@
"lint": "eslint . --max-warnings 0"
},
"dependencies": {
"@mdx-js/loader": "^3.1.1",
"@mdx-js/react": "^3.1.1",
"@next/mdx": "^15.5.3",
"@types/mdx": "^2.0.13",
"@usesend/email-editor": "workspace:*",
"@usesend/ui": "workspace:*",
"iconoir-react": "^7.11.0",

View File

@@ -0,0 +1,19 @@
import type { ReactNode } from "react";
import { SiteFooter } from "~/components/SiteFooter";
import { TopNav } from "~/components/TopNav";
export default function UpdateLayout({
children,
}: {
children: ReactNode;
}) {
return (
<main className="min-h-screen bg-background text-foreground">
<TopNav />
<div className="mx-auto w-full max-w-3xl px-6 py-16">
<article className="space-y-8">{children}</article>
</div>
<SiteFooter />
</main>
);
}

View File

@@ -0,0 +1,66 @@
export const metadata = {
title: "September Outage Update | useSend",
description:
"What happened during the September outage, how we responded, and the improvements now in motion.",
alternates: {
canonical: "https://usesend.com/update/september-outage",
},
openGraph: {
title: "September Outage Update | useSend",
description:
"What happened during the September outage, how we responded, and the improvements now in motion.",
type: "article",
url: "https://usesend.com/update/september-outage",
},
twitter: {
card: "summary_large_image",
title: "September Outage Update | useSend",
description:
"What happened during the September outage, how we responded, and the improvements now in motion.",
},
};
# September Outage Postmortem
On September 17, starting at 11:25 UTC, our emails were not being sent and the outage lasted for almost 10 hours until 21:00 UTC. No emails were sent during this time.
## What happened
Our Amazon SES sending was temporarily paused after compliance signals indicated potential spam characteristics. The initial feedback suggested that some marketing emails might not meet common anti-spam standards (for example, missing unsubscribe links).
## Timeline tldr; (UTC)
- **11:25** - Received an email that our account's sending was paused without prior warning.
- **11:43** - Identified the problematic account, blocked it, and replied to AWS. Paused new signups as well.
- **13:00** - No reply yet, so created a separate escalated support case to get on a call.
- **14:00** - Initial response appeared to interpret us as the sender of the flagged emails; sending was not yet resumed.
- **14:11** - Clarified our product offering again, noting we had blocked the account and paused signups. Shared the useSend site, GitHub, etc.
- **15:53** - Similar response; only valid point is that some marketing emails lacked an unsubscribe link.
- **17:40** - Shipped a change making an unsubscribe link mandatory for marketing emails; shared details on the fix and existing rate limits.
- **18:36** - AWS still not clear with my product and suggestions included adding a CAPTCHA to a form (not applicable to our current flow).
- **19:18** - Re-explained the product and requested senior review for clearer alignment.
- **19:45** - AWS informed that the case would be reviewed within 2-3 business days.
- **19:48** - Requested expedited review due to user impact.
- **21:00** - Finally a valid response with actual steps to improve the product and resumed the account.
## Why
The pause highlighted areas where we can be more diligent about what gets sent through useSend and ensure alignment with SES guidelines and broader email standards.
## What's done till now
- Added a waitlist on signup; we'll screen users before enabling sending.
- Made the unsubscribe link mandatory in the marketing email editor.
- Focusing on users sending transactional and product emails for now.
## Long-term improvements
- More monitoring and pre-send checks (including email screening).
- Double opt-in for contacts.
- A backup SES account to improve resilience.
- Considering a BYO SES option, with useSend managing it for a flat fee.
- Exploring a move to a self-hosted email server (Hard but will try my best).
## To my users
Thank you for being patient and supporting during this time. I'll do a better job in the future to avoid such issues. If you have any suggestions, please do send them in discord or [koushik@usesend.com](mailto:koushik@usesend.com).

View File

@@ -0,0 +1,36 @@
import type { MDXComponents } from "mdx/types";
const components = {
h1: ({ children }) => (
<h1 className="text-3xl font-semibold tracking-wide font-sans text-primary">
{children}
</h1>
),
h2: ({ children }) => (
<h2 className="text-xl font-semibold tracking-wide font-sans text-primary">
{children}
</h2>
),
h3: ({ children }) => (
<h3 className="text-lg font-medium tracking-wide font-sans">{children}</h3>
),
p: ({ children }) => (
<p className="text-base font-normal tracking-wide leading-relaxed font-sans">
{children}
</p>
),
ul: ({ children }) => (
<ul className="list-disc list-inside font-sans pl-4 space-y-1">
{children}
</ul>
),
a: ({ children, href }) => (
<a href={href} className=" text-primary-light">
{children}
</a>
),
} satisfies MDXComponents;
export function useMDXComponents(): MDXComponents {
return components;
}

1120
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff