diff --git a/packages/backend/convex/agentJobs.ts b/packages/backend/convex/agentJobs.ts index 54a79af..4cdbdce 100644 --- a/packages/backend/convex/agentJobs.ts +++ b/packages/backend/convex/agentJobs.ts @@ -1218,7 +1218,8 @@ export const recoverStaleJobs = internalMutation({ .withIndex('by_status', (q) => q.eq('status', status)) .collect(); for (const job of jobs) { - if (isTerminalJob(job)) continue; + if (isTerminalStatus(job.status) || job.completedAt !== undefined) + continue; const last = job.lastHeartbeatAt ?? job.claimedAt ?? job.createdAt; if (last > threshold) continue; const now = Date.now(); diff --git a/packages/backend/tests/unit/recover-stale-jobs.test.ts b/packages/backend/tests/unit/recover-stale-jobs.test.ts index f253add..02b2944 100644 --- a/packages/backend/tests/unit/recover-stale-jobs.test.ts +++ b/packages/backend/tests/unit/recover-stale-jobs.test.ts @@ -161,6 +161,33 @@ describe('recoverStaleJobs', () => { expect(state.thread?.resolvedAt).toBe(resolvedAt); }); + test('reaps an abandoned-stop job that never completed', async () => { + const t = convexTest(schema, modules); + // The user hit "Stop workspace" (markWorkspaceStopped) while the job was + // still running: workspaceStatus flips to 'stopped' but job.status, + // completedAt, and the thread are never finalized. The heartbeat then goes + // stale, so the cron must still reap this job. + const ids = await seedJob(t, { + status: 'running', + workspaceStatus: 'stopped', + // no completedAt: this is NOT a resolved maintenance job + threadStatus: 'running', + claimedAt: Date.now() - 10 * 60 * 1000, + lastHeartbeatAt: Date.now() - 10 * 60 * 1000, + }); + + const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {}); + const state = await readState(t, ids); + + expect(result).toEqual({ recovered: 1 }); + expect(state.job?.status).toBe('timed_out'); + expect(state.job?.workspaceStatus).toBe('expired'); + expect(state.job?.completedAt).toBeTypeOf('number'); + expect(state.request?.status).toBe('failed'); + expect(state.thread?.status).toBe('failed'); + expect(state.thread?.resolvedAt).toBeTypeOf('number'); + }); + test('leaves a running job with a fresh heartbeat untouched', async () => { const t = convexTest(schema, modules); const ids = await seedJob(t, { lastHeartbeatAt: Date.now() });