fix(agent-jobs): narrow recovery-cron guard so abandoned stops still recover
This commit is contained in:
@@ -1218,7 +1218,8 @@ export const recoverStaleJobs = internalMutation({
|
|||||||
.withIndex('by_status', (q) => q.eq('status', status))
|
.withIndex('by_status', (q) => q.eq('status', status))
|
||||||
.collect();
|
.collect();
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
if (isTerminalJob(job)) continue;
|
if (isTerminalStatus(job.status) || job.completedAt !== undefined)
|
||||||
|
continue;
|
||||||
const last = job.lastHeartbeatAt ?? job.claimedAt ?? job.createdAt;
|
const last = job.lastHeartbeatAt ?? job.claimedAt ?? job.createdAt;
|
||||||
if (last > threshold) continue;
|
if (last > threshold) continue;
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|||||||
@@ -161,6 +161,33 @@ describe('recoverStaleJobs', () => {
|
|||||||
expect(state.thread?.resolvedAt).toBe(resolvedAt);
|
expect(state.thread?.resolvedAt).toBe(resolvedAt);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('reaps an abandoned-stop job that never completed', async () => {
|
||||||
|
const t = convexTest(schema, modules);
|
||||||
|
// The user hit "Stop workspace" (markWorkspaceStopped) while the job was
|
||||||
|
// still running: workspaceStatus flips to 'stopped' but job.status,
|
||||||
|
// completedAt, and the thread are never finalized. The heartbeat then goes
|
||||||
|
// stale, so the cron must still reap this job.
|
||||||
|
const ids = await seedJob(t, {
|
||||||
|
status: 'running',
|
||||||
|
workspaceStatus: 'stopped',
|
||||||
|
// no completedAt: this is NOT a resolved maintenance job
|
||||||
|
threadStatus: 'running',
|
||||||
|
claimedAt: Date.now() - 10 * 60 * 1000,
|
||||||
|
lastHeartbeatAt: Date.now() - 10 * 60 * 1000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await t.mutation(internal.agentJobs.recoverStaleJobs, {});
|
||||||
|
const state = await readState(t, ids);
|
||||||
|
|
||||||
|
expect(result).toEqual({ recovered: 1 });
|
||||||
|
expect(state.job?.status).toBe('timed_out');
|
||||||
|
expect(state.job?.workspaceStatus).toBe('expired');
|
||||||
|
expect(state.job?.completedAt).toBeTypeOf('number');
|
||||||
|
expect(state.request?.status).toBe('failed');
|
||||||
|
expect(state.thread?.status).toBe('failed');
|
||||||
|
expect(state.thread?.resolvedAt).toBeTypeOf('number');
|
||||||
|
});
|
||||||
|
|
||||||
test('leaves a running job with a fresh heartbeat untouched', async () => {
|
test('leaves a running job with a fresh heartbeat untouched', async () => {
|
||||||
const t = convexTest(schema, modules);
|
const t = convexTest(schema, modules);
|
||||||
const ids = await seedJob(t, { lastHeartbeatAt: Date.now() });
|
const ids = await seedJob(t, { lastHeartbeatAt: Date.now() });
|
||||||
|
|||||||
Reference in New Issue
Block a user