fix(agent-jobs): add heartbeat cancel and stale recovery

packages/backend/convex/agentJobs.ts

@@ -1201,6 +1201,52 @@ export const failClaimInternal = internalMutation({
   },
 });
 
+export const recoverStaleJobs = internalMutation({
+  args: { staleMs: v.optional(v.number()) },
+  handler: async (ctx, { staleMs }) => {
+    const threshold = Date.now() - (staleMs ?? 3 * 60 * 1000);
+    const activeStatuses = [
+      'claimed',
+      'preparing',
+      'running',
+      'checks_running',
+    ] as const;
+    let recovered = 0;
+    for (const status of activeStatuses) {
+      const jobs = await ctx.db
+        .query('agentJobs')
+        .withIndex('by_status', (q) => q.eq('status', status))
+        .collect();
+      for (const job of jobs) {
+        const last = job.lastHeartbeatAt ?? job.claimedAt ?? job.createdAt;
+        if (last > threshold) continue;
+        const now = Date.now();
+        await ctx.db.patch(job._id, {
+          status: 'timed_out',
+          error:
+            job.error ??
+            'The worker stopped reporting progress; the job timed out.',
+          completedAt: now,
+          updatedAt: now,
+        });
+        await ctx.db.patch(job.agentRequestId, {
+          status: 'failed',
+          updatedAt: now,
+        });
+        if (job.threadId) {
+          await ctx.db.patch(job.threadId, {
+            status: 'failed',
+            updatedAt: now,
+            resolvedAt: now,
+          });
+        }
+        recovered += 1;
+      }
+    }
+    return { recovered };
+  },
+});
+
 export const updateStatus = mutation({
   args: {
     workerToken: v.string(),
@@ -1471,7 +1517,7 @@ export const heartbeatWorkspace = mutation({
       lastHeartbeatAt: Date.now(),
       updatedAt: Date.now(),
     });
-    return { success: true };
+    return { success: true, cancelRequested: job.status === 'cancelled' };
   },
 });
 

packages/backend/convex/crons.ts

@@ -12,6 +12,12 @@ crons.interval(
     limit: 10,
   },
 );
+crons.interval(
+  'Recover stale agent jobs',
+  { minutes: 1 },
+  internal.agentJobs.recoverStaleJobs,
+  {},
+);
 /* Example cron jobs
 crons.cron(
   // Run at 7:00 AM CST / 8:00 AM CDT