fix(worker): make cancellation teardown failure-safe

apps/agent-worker/tests/unit/codex-process.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, test, vi } from 'vitest';
+
+import {
+  abortManagedCodexProcess,
+  managedCodexCommand,
+} from '../../src/codex-process';
+
+describe('managed Codex process', () => {
+  test('launches Codex in a job-specific process session', () => {
+    const command = managedCodexCommand({
+      jobId: 'job-1',
+      pidFile: '/home/alice/.codex/job-1.pid',
+      command: ['codex', 'exec', '--json', 'fix it'],
+    });
+
+    expect(command.slice(0, 2)).toEqual(['python3', '-c']);
+    expect(command).toContain('/home/alice/.codex/job-1.pid');
+    expect(command).toContain('job-1');
+    expect(command.slice(-4)).toEqual(['codex', 'exec', '--json', 'fix it']);
+  });
+
+  test('invokes an isolated in-box kill for the matching job', async () => {
+    const execute = vi.fn().mockResolvedValue({ exitCode: 0, output: '' });
+
+    await abortManagedCodexProcess({
+      jobId: 'job-1',
+      pidFile: '/home/alice/.codex/job-1.pid',
+      execute,
+    });
+
+    expect(execute).toHaveBeenCalledOnce();
+    const [command] = execute.mock.calls[0] as [string[]];
+    expect(command.slice(0, 2)).toEqual(['python3', '-c']);
+    expect(command).toContain('/home/alice/.codex/job-1.pid');
+    expect(command).toContain('job-1');
+  });
+});

apps/agent-worker/tests/unit/heartbeat-controller.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+
+import { createHeartbeatController } from '../../src/heartbeat-controller';
+
+describe('heartbeat controller', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  test('does not overlap ticks while a heartbeat is in flight', async () => {
+    let resolveHeartbeat!: (value: { cancelRequested: boolean }) => void;
+    const heartbeat = vi.fn(
+      () =>
+        new Promise<{ cancelRequested: boolean }>((resolve) => {
+          resolveHeartbeat = resolve;
+        }),
+    );
+    const controller = createHeartbeatController({
+      intervalMs: 30_000,
+      heartbeat,
+      onCancel: vi.fn(),
+      onError: vi.fn(),
+    });
+
+    controller.start('job-1');
+    await vi.advanceTimersByTimeAsync(90_000);
+    expect(heartbeat).toHaveBeenCalledOnce();
+
+    resolveHeartbeat({ cancelRequested: false });
+    await Promise.resolve();
+    await vi.advanceTimersByTimeAsync(30_000);
+    expect(heartbeat).toHaveBeenCalledTimes(2);
+    controller.stop('job-1');
+  });
+
+  test('removes the timer before awaiting cancellation teardown', async () => {
+    let finishCancel!: () => void;
+    const onCancel = vi.fn(
+      () =>
+        new Promise<void>((resolve) => {
+          finishCancel = resolve;
+        }),
+    );
+    const heartbeat = vi.fn().mockResolvedValue({ cancelRequested: true });
+    const controller = createHeartbeatController({
+      intervalMs: 30_000,
+      heartbeat,
+      onCancel,
+      onError: vi.fn(),
+    });
+
+    controller.start('job-1');
+    await vi.advanceTimersByTimeAsync(30_000);
+    expect(onCancel).toHaveBeenCalledOnce();
+    expect(controller.has('job-1')).toBe(false);
+
+    await vi.advanceTimersByTimeAsync(90_000);
+    expect(heartbeat).toHaveBeenCalledOnce();
+    finishCancel();
+  });
+});

apps/agent-worker/tests/unit/workspace-teardown.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, test, vi } from 'vitest';
+
+import { abortManagedCodexProcess } from '../../src/codex-process';
+import { teardownWorkspace } from '../../src/workspace-teardown';
+
+describe('workspace teardown', () => {
+  test('event failure does not skip abort, stop, removal, or release', async () => {
+    const calls: string[] = [];
+    const record = (name: string) => vi.fn(() => calls.push(name));
+
+    const result = await teardownWorkspace({
+      stopHeartbeat: record('heartbeat'),
+      removeActive: record('remove'),
+      appendEvent: vi.fn().mockRejectedValue(new Error('event failed')),
+      abortAgent: record('abort'),
+      markStopped: vi.fn().mockRejectedValue(new Error('mutation failed')),
+      closeAgent: record('close'),
+      stopContainer: vi.fn().mockRejectedValue(new Error('container failed')),
+      release: record('release'),
+    });
+
+    expect(calls).toEqual(['heartbeat', 'remove', 'abort', 'close', 'release']);
+    expect(result.errors).toHaveLength(3);
+  });
+
+  test('removes active state before asynchronous cancellation work', async () => {
+    const active = new Set(['job-1']);
+    let activeDuringAbort = true;
+
+    await teardownWorkspace({
+      stopHeartbeat: vi.fn(),
+      removeActive: () => active.delete('job-1'),
+      abortAgent: () => {
+        activeDuringAbort = active.has('job-1');
+      },
+      release: vi.fn(),
+    });
+
+    expect(activeDuringAbort).toBe(false);
+    expect(active.has('job-1')).toBe(false);
+  });
+
+  test('Codex cancellation invokes isolated kill and leaves no active workspace', async () => {
+    const active = new Set(['job-1']);
+    const execute = vi.fn().mockResolvedValue({ exitCode: 0, output: '' });
+
+    await teardownWorkspace({
+      stopHeartbeat: vi.fn(),
+      removeActive: () => active.delete('job-1'),
+      abortAgent: async () =>
+        await abortManagedCodexProcess({
+          jobId: 'job-1',
+          pidFile: '/home/alice/.codex/job-1.pid',
+          execute,
+        }),
+      release: vi.fn(),
+    });
+
+    expect(execute).toHaveBeenCalledOnce();
+    expect(active.has('job-1')).toBe(false);
+  });
+});