fix(agent-jobs): narrow recovery-cron guard so abandoned stops still recover

This commit is contained in:
Gabriel Brown
2026-07-10 18:34:15 -04:00
parent 38431155b2
commit 613a6c8a2d
2 changed files with 29 additions and 1 deletions
+2 -1
View File
@@ -1218,7 +1218,8 @@ export const recoverStaleJobs = internalMutation({
.withIndex('by_status', (q) => q.eq('status', status)) .withIndex('by_status', (q) => q.eq('status', status))
.collect(); .collect();
for (const job of jobs) { for (const job of jobs) {
if (isTerminalJob(job)) continue; if (isTerminalStatus(job.status) || job.completedAt !== undefined)
continue;
const last = job.lastHeartbeatAt ?? job.claimedAt ?? job.createdAt; const last = job.lastHeartbeatAt ?? job.claimedAt ?? job.createdAt;
if (last > threshold) continue; if (last > threshold) continue;
const now = Date.now(); const now = Date.now();
@@ -161,6 +161,33 @@ describe('recoverStaleJobs', () => {
expect(state.thread?.resolvedAt).toBe(resolvedAt); expect(state.thread?.resolvedAt).toBe(resolvedAt);
}); });
test('reaps an abandoned-stop job that never completed', async () => {
const t = convexTest(schema, modules);
// The user hit "Stop workspace" (markWorkspaceStopped) while the job was
// still running: workspaceStatus flips to 'stopped' but job.status,
// completedAt, and the thread are never finalized. The heartbeat then goes
// stale, so the cron must still reap this job.
const ids = await seedJob(t, {
status: 'running',
workspaceStatus: 'stopped',
// no completedAt: this is NOT a resolved maintenance job
threadStatus: 'running',
claimedAt: Date.now() - 10 * 60 * 1000,
lastHeartbeatAt: Date.now() - 10 * 60 * 1000,
});
const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
const state = await readState(t, ids);
expect(result).toEqual({ recovered: 1 });
expect(state.job?.status).toBe('timed_out');
expect(state.job?.workspaceStatus).toBe('expired');
expect(state.job?.completedAt).toBeTypeOf('number');
expect(state.request?.status).toBe('failed');
expect(state.thread?.status).toBe('failed');
expect(state.thread?.resolvedAt).toBeTypeOf('number');
});
test('leaves a running job with a fresh heartbeat untouched', async () => { test('leaves a running job with a fresh heartbeat untouched', async () => {
const t = convexTest(schema, modules); const t = convexTest(schema, modules);
const ids = await seedJob(t, { lastHeartbeatAt: Date.now() }); const ids = await seedJob(t, { lastHeartbeatAt: Date.now() });