fix(worker): make cancellation teardown failure-safe

packages/backend/tests/unit/recover-stale-jobs.test.ts

@@ -1,5 +1,5 @@
 import { convexTest } from 'convex-test';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 
 import type { Id } from '../../convex/_generated/dataModel.js';
 import { internal } from '../../convex/_generated/api.js';
@@ -7,9 +7,15 @@ import schema from '../../convex/schema';
 
 const modules = import.meta.glob('../../convex/**/*.*s');
 
-const seedRunningJob = async (
+type ActiveStatus = 'claimed' | 'preparing' | 'running' | 'checks_running';
+
+const seedJob = async (
   t: ReturnType<typeof convexTest>,
-  lastHeartbeatAt: number,
+  args: {
+    status?: ActiveStatus;
+    claimedAt?: number;
+    lastHeartbeatAt?: number;
+  },
 ) =>
   await t.mutation(async (ctx) => {
     const now = Date.now();
@@ -60,7 +66,7 @@ const seedRunningJob = async (
       ownerId,
       agentRequestId: requestId,
       threadId,
-      status: 'running',
+      status: args.status ?? 'running',
       prompt: 'Recover the worker job',
       runtime: 'opencode',
       workspaceStatus: 'active',
@@ -75,8 +81,10 @@ const seedRunningJob = async (
       model: 'gpt-5.1-codex',
       reasoningEffort: 'medium',
       claimedBy: 'w1',
-      claimedAt: now,
-      lastHeartbeatAt,
+      claimedAt: args.claimedAt ?? now,
+      ...(args.lastHeartbeatAt === undefined
+        ? {}
+        : { lastHeartbeatAt: args.lastHeartbeatAt }),
       createdAt: now,
       updatedAt: now,
     });
@@ -102,7 +110,9 @@ const readState = async (
 describe('recoverStaleJobs', () => {
   test('times out a running job with a stale heartbeat', async () => {
     const t = convexTest(schema, modules);
-    const ids = await seedRunningJob(t, Date.now() - 10 * 60 * 1000);
+    const ids = await seedJob(t, {
+      lastHeartbeatAt: Date.now() - 10 * 60 * 1000,
+    });
 
     const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
     const state = await readState(t, ids);
@@ -117,7 +127,7 @@ describe('recoverStaleJobs', () => {
 
   test('leaves a running job with a fresh heartbeat untouched', async () => {
     const t = convexTest(schema, modules);
-    const ids = await seedRunningJob(t, Date.now());
+    const ids = await seedJob(t, { lastHeartbeatAt: Date.now() });
 
     const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
     const state = await readState(t, ids);
@@ -127,4 +137,57 @@ describe('recoverStaleJobs', () => {
     expect(state.request?.status).toBe('running');
     expect(state.thread?.status).toBe('running');
   });
+
+  test('uses claimedAt when a claimed job has no heartbeat', async () => {
+    const t = convexTest(schema, modules);
+    const ids = await seedJob(t, {
+      status: 'claimed',
+      claimedAt: Date.now() - 10 * 60 * 1000,
+    });
+
+    const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
+    const state = await readState(t, ids);
+
+    expect(result).toEqual({ recovered: 1 });
+    expect(state.job?.status).toBe('timed_out');
+  });
+
+  test('times out a heartbeat exactly at the default threshold', async () => {
+    vi.useFakeTimers();
+    try {
+      const now = new Date('2026-07-10T12:00:00.000Z');
+      vi.setSystemTime(now);
+      const t = convexTest(schema, modules);
+      const ids = await seedJob(t, {
+        lastHeartbeatAt: now.getTime() - 3 * 60 * 1000,
+      });
+
+      const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
+      const state = await readState(t, ids);
+
+      expect(result).toEqual({ recovered: 1 });
+      expect(state.job?.status).toBe('timed_out');
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  test.each<ActiveStatus>([
+    'claimed',
+    'preparing',
+    'running',
+    'checks_running',
+  ])('recovers stale %s jobs', async (status) => {
+    const t = convexTest(schema, modules);
+    const ids = await seedJob(t, {
+      status,
+      lastHeartbeatAt: Date.now() - 10 * 60 * 1000,
+    });
+
+    const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
+    const state = await readState(t, ids);
+
+    expect(result).toEqual({ recovered: 1 });
+    expect(state.job?.status).toBe('timed_out');
+  });
 });