Add agent workflows & stuff
This commit is contained in:
Gabriel Brown
@@ -0,0 +1,11 @@
|
||||
import { defineConfig } from 'eslint/config';
|
||||
|
||||
import { baseConfig } from '@spoon/eslint-config/base';
|
||||
|
||||
export default defineConfig(baseConfig, {
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
tsconfigRootDir: import.meta.dirname,
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "@spoon/agent-worker",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "bun with-env src/index.ts",
|
||||
"start": "bun src/index.ts",
|
||||
"format": "prettier --check . --ignore-path ../../.gitignore",
|
||||
"lint": "eslint --flag unstable_native_nodejs_ts_config",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test:unit": "vitest run --project unit --passWithNoTests",
|
||||
"test:integration": "vitest run --project integration --passWithNoTests",
|
||||
"test:component": "vitest run --project component --passWithNoTests",
|
||||
"with-env": "sh ../../scripts/with-env ${INFISICAL_ENV:-dev} --"
|
||||
},
|
||||
"dependencies": {
|
||||
"@octokit/auth-app": "^8.2.0",
|
||||
"@octokit/rest": "^22.0.1",
|
||||
"@openai/agents": "latest",
|
||||
"convex": "catalog:convex",
|
||||
"execa": "latest",
|
||||
"openai": "^6.44.0",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@spoon/eslint-config": "workspace:*",
|
||||
"@spoon/prettier-config": "workspace:*",
|
||||
"@spoon/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"eslint": "catalog:",
|
||||
"prettier": "catalog:",
|
||||
"typescript": "catalog:",
|
||||
"vitest": "catalog:test"
|
||||
},
|
||||
"prettier": "@spoon/prettier-config"
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { execa } from 'execa';
|
||||
import OpenAI from 'openai';
|
||||
|
||||
const editSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
summary: { type: 'string' },
|
||||
files: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
path: { type: 'string' },
|
||||
content: { type: 'string' },
|
||||
},
|
||||
required: ['path', 'content'],
|
||||
},
|
||||
},
|
||||
commands: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
},
|
||||
limitations: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
},
|
||||
},
|
||||
required: ['summary', 'files', 'commands', 'limitations'],
|
||||
} as const;
|
||||
|
||||
type AgentEdit = {
|
||||
summary: string;
|
||||
files: { path: string; content: string }[];
|
||||
commands: string[];
|
||||
limitations: string[];
|
||||
};
|
||||
|
||||
const maxContextFiles = 40;
|
||||
const maxFileBytes = 12_000;
|
||||
|
||||
const safeContextFile = (file: string) =>
|
||||
!file.includes('node_modules/') &&
|
||||
!file.includes('.git/') &&
|
||||
!file.includes('dist/') &&
|
||||
!file.includes('build/') &&
|
||||
!file.includes('.next/') &&
|
||||
!file.endsWith('.lock') &&
|
||||
!file.endsWith('.png') &&
|
||||
!file.endsWith('.jpg') &&
|
||||
!file.endsWith('.jpeg') &&
|
||||
!file.endsWith('.webp') &&
|
||||
!file.endsWith('.gif') &&
|
||||
!file.endsWith('.pdf');
|
||||
|
||||
const listFiles = async (repoDir: string) => {
|
||||
const result = await execa('git', ['ls-files'], {
|
||||
cwd: repoDir,
|
||||
all: true,
|
||||
reject: false,
|
||||
});
|
||||
return result.all
|
||||
.split('\n')
|
||||
.map((file) => file.trim())
|
||||
.filter(Boolean)
|
||||
.filter(safeContextFile);
|
||||
};
|
||||
|
||||
const chooseContextFiles = (files: string[], prompt: string) => {
|
||||
const promptWords = new Set(
|
||||
prompt
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter((word) => word.length > 3),
|
||||
);
|
||||
const scored = files.map((file) => {
|
||||
const lower = file.toLowerCase();
|
||||
const score = [...promptWords].reduce(
|
||||
(sum, word) => sum + (lower.includes(word) ? 2 : 0),
|
||||
/(readme|package\.json|auth|env|config|route|provider)/i.exec(file)
|
||||
? 3
|
||||
: 0,
|
||||
);
|
||||
return { file, score };
|
||||
});
|
||||
return scored
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, maxContextFiles)
|
||||
.map((item) => item.file);
|
||||
};
|
||||
|
||||
const readContext = async (repoDir: string, files: string[]) => {
|
||||
const chunks = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = await readFile(path.join(repoDir, file), 'utf8');
|
||||
chunks.push({
|
||||
path: file,
|
||||
content:
|
||||
content.length > maxFileBytes
|
||||
? `${content.slice(0, maxFileBytes)}\n[truncated]`
|
||||
: content,
|
||||
});
|
||||
} catch {
|
||||
// Ignore files that disappeared while context was being gathered.
|
||||
}
|
||||
}
|
||||
return chunks;
|
||||
};
|
||||
|
||||
const parseEdit = (value: string): AgentEdit => {
|
||||
const parsed = JSON.parse(value) as AgentEdit;
|
||||
if (!Array.isArray(parsed.files)) {
|
||||
throw new Error('OpenAI returned an edit without a files array.');
|
||||
}
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const safePath = (repoDir: string, filePath: string) => {
|
||||
const resolved = path.resolve(repoDir, filePath);
|
||||
if (!resolved.startsWith(path.resolve(repoDir))) {
|
||||
throw new Error(`Refusing to write outside the repository: ${filePath}`);
|
||||
}
|
||||
return resolved;
|
||||
};
|
||||
|
||||
export const runOpenAiEdit = async (args: {
|
||||
repoDir: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
reasoningEffort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
||||
prompt: string;
|
||||
secretNames: string[];
|
||||
spoonName: string;
|
||||
upstreamFullName: string;
|
||||
forkFullName: string;
|
||||
}) => {
|
||||
const files = await listFiles(args.repoDir);
|
||||
const selectedFiles = chooseContextFiles(files, args.prompt);
|
||||
const contextFiles = await readContext(args.repoDir, selectedFiles);
|
||||
const response = await new OpenAI({ apiKey: args.apiKey }).responses.create({
|
||||
model: args.model,
|
||||
store: false,
|
||||
reasoning:
|
||||
args.reasoningEffort === 'none'
|
||||
? undefined
|
||||
: { effort: args.reasoningEffort },
|
||||
input: [
|
||||
{
|
||||
role: 'system',
|
||||
content:
|
||||
'You are a conservative coding agent working in a fork. Return complete replacement contents only for files that must change. Keep the diff minimal. Do not include secrets. Do not claim commands passed unless they are listed for the worker to run. If the context is insufficient, make the safest small change and describe limitations.',
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: JSON.stringify(
|
||||
{
|
||||
task: args.prompt,
|
||||
spoon: args.spoonName,
|
||||
upstream: args.upstreamFullName,
|
||||
fork: args.forkFullName,
|
||||
availableSecretNames: args.secretNames,
|
||||
repositoryFiles: files.slice(0, 500),
|
||||
contextFiles,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
},
|
||||
],
|
||||
text: {
|
||||
format: {
|
||||
type: 'json_schema',
|
||||
name: 'spoon_agent_file_edits',
|
||||
strict: true,
|
||||
schema: editSchema,
|
||||
},
|
||||
},
|
||||
});
|
||||
const edit = parseEdit(response.output_text);
|
||||
for (const file of edit.files) {
|
||||
const target = safePath(args.repoDir, file.path);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
await writeFile(target, file.content);
|
||||
}
|
||||
return edit;
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
const intEnv = (name: string, fallback: number) => {
|
||||
const value = process.env[name];
|
||||
if (!value) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
};
|
||||
|
||||
const requiredEnv = (name: string) => {
|
||||
const value = process.env[name]?.trim();
|
||||
if (!value) throw new Error(`${name} is required.`);
|
||||
return value;
|
||||
};
|
||||
|
||||
export const env = {
|
||||
convexUrl:
|
||||
process.env.NEXT_PUBLIC_CONVEX_URL?.trim() ??
|
||||
process.env.CONVEX_SELF_HOSTED_URL?.trim() ??
|
||||
'http://localhost:3210',
|
||||
workerToken: requiredEnv('SPOON_WORKER_TOKEN'),
|
||||
workerId: process.env.SPOON_AGENT_WORKER_ID?.trim() ?? 'local-worker',
|
||||
runtime: process.env.SPOON_AGENT_RUNTIME?.trim() ?? 'docker',
|
||||
jobImage:
|
||||
process.env.SPOON_AGENT_JOB_IMAGE?.trim() ?? 'spoon-agent-job:latest',
|
||||
workdir: process.env.SPOON_AGENT_WORKDIR?.trim() ?? '.local/agent-work',
|
||||
network: process.env.SPOON_AGENT_NETWORK?.trim(),
|
||||
pollMs: intEnv('SPOON_AGENT_POLL_MS', 5_000),
|
||||
maxConcurrentJobs: intEnv('SPOON_AGENT_MAX_CONCURRENT_JOBS', 1),
|
||||
jobTimeoutMs: intEnv('SPOON_AGENT_JOB_TIMEOUT_MS', 1_800_000),
|
||||
githubAppId: requiredEnv('GITHUB_APP_ID'),
|
||||
githubPrivateKey: requiredEnv('GITHUB_APP_PRIVATE_KEY').replaceAll(
|
||||
'\\n',
|
||||
'\n',
|
||||
),
|
||||
};
|
||||
@@ -0,0 +1,144 @@
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { execa } from 'execa';
|
||||
|
||||
export type RunOptions = {
|
||||
cwd: string;
|
||||
env?: Record<string, string>;
|
||||
redact: (value: string) => string;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
export const run = async (
|
||||
command: string,
|
||||
args: string[],
|
||||
options: RunOptions,
|
||||
) => {
|
||||
const result = await execa(command, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
all: true,
|
||||
reject: false,
|
||||
timeout: options.timeoutMs,
|
||||
});
|
||||
return {
|
||||
exitCode: result.exitCode ?? 0,
|
||||
output: options.redact(result.all),
|
||||
};
|
||||
};
|
||||
|
||||
export const cloneRepository = async (args: {
|
||||
workdir: string;
|
||||
token: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
baseBranch: string;
|
||||
workBranch: string;
|
||||
redact: (value: string) => string;
|
||||
timeoutMs: number;
|
||||
}) => {
|
||||
await mkdir(args.workdir, { recursive: true });
|
||||
const repoUrl = `https://x-access-token:${args.token}@github.com/${args.owner}/${args.repo}.git`;
|
||||
const clone = await run(
|
||||
'git',
|
||||
['clone', '--branch', args.baseBranch, '--single-branch', repoUrl, 'repo'],
|
||||
{
|
||||
cwd: args.workdir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
},
|
||||
);
|
||||
if (clone.exitCode !== 0) {
|
||||
throw new Error(`git clone failed:\n${clone.output}`);
|
||||
}
|
||||
const repoDir = path.join(args.workdir, 'repo');
|
||||
const checkout = await run('git', ['checkout', '-b', args.workBranch], {
|
||||
cwd: repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
});
|
||||
if (checkout.exitCode !== 0) {
|
||||
throw new Error(`git checkout failed:\n${checkout.output}`);
|
||||
}
|
||||
return repoDir;
|
||||
};
|
||||
|
||||
export const commitAndPush = async (args: {
|
||||
repoDir: string;
|
||||
workBranch: string;
|
||||
message: string;
|
||||
redact: (value: string) => string;
|
||||
timeoutMs: number;
|
||||
}) => {
|
||||
await run('git', ['config', 'user.name', 'Spoon Agent'], {
|
||||
cwd: args.repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
});
|
||||
await run(
|
||||
'git',
|
||||
['config', 'user.email', 'spoon-agent@users.noreply.github.com'],
|
||||
{
|
||||
cwd: args.repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
},
|
||||
);
|
||||
await run('git', ['add', '-A'], {
|
||||
cwd: args.repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
});
|
||||
const commit = await run('git', ['commit', '-m', args.message], {
|
||||
cwd: args.repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
});
|
||||
if (commit.exitCode !== 0) {
|
||||
throw new Error(`git commit failed:\n${commit.output}`);
|
||||
}
|
||||
const sha = await run('git', ['rev-parse', 'HEAD'], {
|
||||
cwd: args.repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
});
|
||||
const push = await run('git', ['push', 'origin', args.workBranch], {
|
||||
cwd: args.repoDir,
|
||||
redact: args.redact,
|
||||
timeoutMs: args.timeoutMs,
|
||||
});
|
||||
if (push.exitCode !== 0) {
|
||||
throw new Error(`git push failed:\n${push.output}`);
|
||||
}
|
||||
return sha.output.trim();
|
||||
};
|
||||
|
||||
export const getDiff = async (
|
||||
repoDir: string,
|
||||
redact: (value: string) => string,
|
||||
) =>
|
||||
await run('git', ['diff', '--cached', '--', '.'], {
|
||||
cwd: repoDir,
|
||||
redact,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
|
||||
export const getWorktreeDiff = async (
|
||||
repoDir: string,
|
||||
redact: (value: string) => string,
|
||||
) =>
|
||||
await run('git', ['diff', '--', '.'], {
|
||||
cwd: repoDir,
|
||||
redact,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
|
||||
export const getStatus = async (
|
||||
repoDir: string,
|
||||
redact: (value: string) => string,
|
||||
) =>
|
||||
await run('git', ['status', '--short'], {
|
||||
cwd: repoDir,
|
||||
redact,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { createAppAuth } from '@octokit/auth-app';
|
||||
import { Octokit } from '@octokit/rest';
|
||||
|
||||
import { env } from './env';
|
||||
|
||||
export const getInstallationToken = async (installationId: string) => {
|
||||
const auth = createAppAuth({
|
||||
appId: env.githubAppId,
|
||||
privateKey: env.githubPrivateKey,
|
||||
installationId,
|
||||
});
|
||||
const result = await auth({ type: 'installation' });
|
||||
return result.token;
|
||||
};
|
||||
|
||||
export const getInstallationOctokit = (installationId: string) =>
|
||||
new Octokit({
|
||||
authStrategy: createAppAuth,
|
||||
auth: {
|
||||
appId: env.githubAppId,
|
||||
privateKey: env.githubPrivateKey,
|
||||
installationId,
|
||||
},
|
||||
userAgent: 'Spoon Agent Worker',
|
||||
request: {
|
||||
headers: {
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export const openDraftPullRequest = async (args: {
|
||||
installationId: string;
|
||||
forkOwner: string;
|
||||
forkRepo: string;
|
||||
baseBranch: string;
|
||||
workBranch: string;
|
||||
title: string;
|
||||
body: string;
|
||||
}) => {
|
||||
const octokit = getInstallationOctokit(args.installationId);
|
||||
const result = await octokit.rest.pulls.create({
|
||||
owner: args.forkOwner,
|
||||
repo: args.forkRepo,
|
||||
base: args.baseBranch,
|
||||
head: args.workBranch,
|
||||
title: args.title,
|
||||
body: args.body,
|
||||
draft: true,
|
||||
});
|
||||
return result.data;
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
import { startWorker } from './worker';
|
||||
|
||||
await startWorker();
|
||||
@@ -0,0 +1,26 @@
|
||||
const secretPatterns = [
|
||||
/ghs_[A-Za-z0-9_]+/g,
|
||||
/github_pat_[A-Za-z0-9_]+/g,
|
||||
/sk-[A-Za-z0-9_-]+/g,
|
||||
/(client_secret|auth_secret|api_key|token|password)=([^ \n\r]+)/gi,
|
||||
];
|
||||
|
||||
export const createRedactor = (values: string[]) => {
|
||||
const secrets = values.filter((value) => value.length >= 3);
|
||||
return (input: string) => {
|
||||
let output = input;
|
||||
for (const secret of secrets) {
|
||||
output = output.split(secret).join('[redacted]');
|
||||
}
|
||||
for (const pattern of secretPatterns) {
|
||||
output = output.replace(pattern, '$1=[redacted]');
|
||||
}
|
||||
return output;
|
||||
};
|
||||
};
|
||||
|
||||
export const truncate = (value: string, maxBytes: number) => {
|
||||
const buffer = Buffer.from(value);
|
||||
if (buffer.byteLength <= maxBytes) return value;
|
||||
return `${buffer.subarray(0, maxBytes).toString('utf8')}\n[truncated]`;
|
||||
};
|
||||
@@ -0,0 +1,45 @@
|
||||
import { execa } from 'execa';
|
||||
|
||||
import { env } from '../env';
|
||||
|
||||
export const runInJobContainer = async (args: {
|
||||
workdir: string;
|
||||
command: string[];
|
||||
environment: Record<string, string>;
|
||||
redact: (value: string) => string;
|
||||
timeoutMs: number;
|
||||
}) => {
|
||||
const envArgs = Object.entries(args.environment).flatMap(([name, value]) => [
|
||||
'-e',
|
||||
`${name}=${value}`,
|
||||
]);
|
||||
const networkArgs = env.network ? ['--network', env.network] : [];
|
||||
const result = await execa(
|
||||
'docker',
|
||||
[
|
||||
'run',
|
||||
'--rm',
|
||||
'--memory',
|
||||
'4g',
|
||||
'--cpus',
|
||||
'2',
|
||||
...networkArgs,
|
||||
...envArgs,
|
||||
'-v',
|
||||
`${args.workdir}:/workspace`,
|
||||
'-w',
|
||||
'/workspace/repo',
|
||||
env.jobImage,
|
||||
...args.command,
|
||||
],
|
||||
{
|
||||
all: true,
|
||||
reject: false,
|
||||
timeout: args.timeoutMs,
|
||||
},
|
||||
);
|
||||
return {
|
||||
exitCode: result.exitCode ?? 0,
|
||||
output: args.redact(result.all),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,423 @@
|
||||
import { access, readFile, rm } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { ConvexHttpClient } from 'convex/browser';
|
||||
|
||||
import type { Id } from '@spoon/backend/convex/_generated/dataModel.js';
|
||||
import { api } from '@spoon/backend/convex/_generated/api.js';
|
||||
|
||||
import { runOpenAiEdit } from './agent';
|
||||
import { env } from './env';
|
||||
import {
|
||||
cloneRepository,
|
||||
commitAndPush,
|
||||
getStatus,
|
||||
getWorktreeDiff,
|
||||
run,
|
||||
} from './git';
|
||||
import { getInstallationToken, openDraftPullRequest } from './github';
|
||||
import { createRedactor, truncate } from './redact';
|
||||
import { runInJobContainer } from './runtime/docker';
|
||||
|
||||
type Claim = {
|
||||
job: {
|
||||
_id: Id<'agentJobs'>;
|
||||
prompt: string;
|
||||
baseBranch: string;
|
||||
workBranch: string;
|
||||
forkOwner: string;
|
||||
forkRepo: string;
|
||||
upstreamOwner: string;
|
||||
upstreamRepo: string;
|
||||
};
|
||||
spoon: { name: string };
|
||||
openai: {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
reasoningEffort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
||||
};
|
||||
github: { installationId?: string };
|
||||
agentSettings?: {
|
||||
installCommand?: string;
|
||||
checkCommand?: string;
|
||||
testCommand?: string;
|
||||
} | null;
|
||||
secrets: { name: string; value: string }[];
|
||||
};
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const client = new ConvexHttpClient(env.convexUrl);
|
||||
|
||||
const appendEvent = async (
|
||||
jobId: Id<'agentJobs'>,
|
||||
level: 'debug' | 'info' | 'warn' | 'error',
|
||||
phase:
|
||||
| 'queued'
|
||||
| 'clone'
|
||||
| 'plan'
|
||||
| 'edit'
|
||||
| 'install'
|
||||
| 'check'
|
||||
| 'test'
|
||||
| 'commit'
|
||||
| 'push'
|
||||
| 'pr'
|
||||
| 'cleanup',
|
||||
message: string,
|
||||
metadata?: string,
|
||||
) =>
|
||||
await client.mutation(api.agentJobs.appendEvent, {
|
||||
workerToken: env.workerToken,
|
||||
workerId: env.workerId,
|
||||
jobId,
|
||||
level,
|
||||
phase,
|
||||
message,
|
||||
metadata,
|
||||
});
|
||||
|
||||
const updateStatus = async (
|
||||
jobId: Id<'agentJobs'>,
|
||||
status:
|
||||
| 'queued'
|
||||
| 'claimed'
|
||||
| 'preparing'
|
||||
| 'running'
|
||||
| 'checks_running'
|
||||
| 'changes_ready'
|
||||
| 'draft_pr_opened'
|
||||
| 'failed'
|
||||
| 'cancelled'
|
||||
| 'timed_out',
|
||||
extra?: { error?: string; summary?: string },
|
||||
) =>
|
||||
await client.mutation(api.agentJobs.updateStatus, {
|
||||
workerToken: env.workerToken,
|
||||
workerId: env.workerId,
|
||||
jobId,
|
||||
status,
|
||||
...extra,
|
||||
});
|
||||
|
||||
const addArtifact = async (args: {
|
||||
jobId: Id<'agentJobs'>;
|
||||
kind: 'plan' | 'diff' | 'test_output' | 'summary' | 'error' | 'pr_body';
|
||||
title: string;
|
||||
content: string;
|
||||
contentType:
|
||||
| 'text/markdown'
|
||||
| 'text/plain'
|
||||
| 'application/json'
|
||||
| 'text/x-diff';
|
||||
}) =>
|
||||
await client.mutation(api.agentJobs.addArtifact, {
|
||||
workerToken: env.workerToken,
|
||||
workerId: env.workerId,
|
||||
...args,
|
||||
});
|
||||
|
||||
const completeWithDraftPr = async (args: {
|
||||
jobId: Id<'agentJobs'>;
|
||||
commitSha: string;
|
||||
pullRequestUrl: string;
|
||||
pullRequestNumber: number;
|
||||
summary: string;
|
||||
}) =>
|
||||
await client.mutation(api.agentJobs.completeWithDraftPr, {
|
||||
workerToken: env.workerToken,
|
||||
workerId: env.workerId,
|
||||
...args,
|
||||
});
|
||||
|
||||
const commandToShell = (command: string) => ['bash', '-lc', command];
|
||||
|
||||
const fileExists = async (filePath: string) => {
|
||||
try {
|
||||
await access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const runProjectCommand = async (args: {
|
||||
command: string;
|
||||
phase: 'install' | 'check' | 'test';
|
||||
claim: Claim;
|
||||
workdir: string;
|
||||
repoDir: string;
|
||||
redact: (value: string) => string;
|
||||
}) => {
|
||||
await appendEvent(args.claim.job._id, 'info', args.phase, args.command);
|
||||
const result =
|
||||
env.runtime === 'docker'
|
||||
? await runInJobContainer({
|
||||
workdir: args.workdir,
|
||||
command: commandToShell(args.command),
|
||||
environment: Object.fromEntries(
|
||||
args.claim.secrets.map((secret) => [secret.name, secret.value]),
|
||||
),
|
||||
redact: args.redact,
|
||||
timeoutMs: env.jobTimeoutMs,
|
||||
})
|
||||
: await run('bash', ['-lc', args.command], {
|
||||
cwd: args.repoDir,
|
||||
env: Object.fromEntries(
|
||||
args.claim.secrets.map((secret) => [secret.name, secret.value]),
|
||||
),
|
||||
redact: args.redact,
|
||||
timeoutMs: env.jobTimeoutMs,
|
||||
});
|
||||
await addArtifact({
|
||||
jobId: args.claim.job._id,
|
||||
kind: args.phase === 'test' ? 'test_output' : 'summary',
|
||||
title: args.command,
|
||||
content: truncate(result.output, 100_000),
|
||||
contentType: 'text/plain',
|
||||
});
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(`${args.command} failed:\n${result.output}`);
|
||||
}
|
||||
};
|
||||
|
||||
const detectPackageCommands = async (
|
||||
repoDir: string,
|
||||
): Promise<{ install?: string; check?: string; test?: string }> => {
|
||||
const packageJsonPath = path.join(repoDir, 'package.json');
|
||||
try {
|
||||
const packageJson = JSON.parse(await readFile(packageJsonPath, 'utf8')) as {
|
||||
scripts?: Record<string, string>;
|
||||
};
|
||||
const scripts = packageJson.scripts ?? {};
|
||||
return {
|
||||
install: (await fileExists(path.join(repoDir, 'bun.lock')))
|
||||
? 'bun install'
|
||||
: (await fileExists(path.join(repoDir, 'pnpm-lock.yaml')))
|
||||
? 'pnpm install'
|
||||
: (await fileExists(path.join(repoDir, 'yarn.lock')))
|
||||
? 'yarn install'
|
||||
: 'npm install',
|
||||
check: scripts.typecheck
|
||||
? 'npm run typecheck'
|
||||
: scripts.lint
|
||||
? 'npm run lint'
|
||||
: undefined,
|
||||
test: scripts.test ? 'npm test' : undefined,
|
||||
};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
const buildPrBody = (args: {
|
||||
prompt: string;
|
||||
summary: string;
|
||||
commands: string[];
|
||||
limitations: string[];
|
||||
}) => `## Spoon agent request
|
||||
|
||||
${args.prompt}
|
||||
|
||||
## Summary
|
||||
|
||||
${args.summary}
|
||||
|
||||
## Validation
|
||||
|
||||
${
|
||||
args.commands.length
|
||||
? args.commands.map((command) => `- \`${command}\``).join('\n')
|
||||
: '- No validation commands were requested by the agent.'
|
||||
}
|
||||
|
||||
## Limitations
|
||||
|
||||
${
|
||||
args.limitations.length
|
||||
? args.limitations.map((item) => `- ${item}`).join('\n')
|
||||
: '- No limitations reported.'
|
||||
}
|
||||
|
||||
Generated by Spoon.`;
|
||||
|
||||
const runClaim = async (claim: Claim) => {
|
||||
const jobId = claim.job._id;
|
||||
const workdir = path.resolve(env.workdir, jobId);
|
||||
const secretValues = [
|
||||
claim.openai.apiKey,
|
||||
...claim.secrets.map((secret) => secret.value),
|
||||
];
|
||||
const redact = createRedactor(secretValues);
|
||||
try {
|
||||
await updateStatus(jobId, 'preparing');
|
||||
await appendEvent(jobId, 'info', 'clone', 'Creating installation token.');
|
||||
if (!claim.github.installationId) {
|
||||
throw new Error('GitHub installation ID is missing.');
|
||||
}
|
||||
const githubToken = await getInstallationToken(claim.github.installationId);
|
||||
const repoDir = await cloneRepository({
|
||||
workdir,
|
||||
token: githubToken,
|
||||
owner: claim.job.forkOwner,
|
||||
repo: claim.job.forkRepo,
|
||||
baseBranch: claim.job.baseBranch,
|
||||
workBranch: claim.job.workBranch,
|
||||
redact,
|
||||
timeoutMs: env.jobTimeoutMs,
|
||||
});
|
||||
await updateStatus(jobId, 'running');
|
||||
await appendEvent(jobId, 'info', 'plan', 'Gathering repo context.');
|
||||
const edit = await runOpenAiEdit({
|
||||
repoDir,
|
||||
apiKey: claim.openai.apiKey,
|
||||
model: claim.openai.model,
|
||||
reasoningEffort: claim.openai.reasoningEffort,
|
||||
prompt: claim.job.prompt,
|
||||
secretNames: claim.secrets.map((secret) => secret.name),
|
||||
spoonName: claim.spoon.name,
|
||||
upstreamFullName: `${claim.job.upstreamOwner}/${claim.job.upstreamRepo}`,
|
||||
forkFullName: `${claim.job.forkOwner}/${claim.job.forkRepo}`,
|
||||
});
|
||||
await addArtifact({
|
||||
jobId,
|
||||
kind: 'plan',
|
||||
title: 'Agent plan',
|
||||
content: edit.summary,
|
||||
contentType: 'text/markdown',
|
||||
});
|
||||
const status = await getStatus(repoDir, redact);
|
||||
if (!status.output.trim()) {
|
||||
throw new Error('No changes produced by the agent.');
|
||||
}
|
||||
const diff = await getWorktreeDiff(repoDir, redact);
|
||||
await addArtifact({
|
||||
jobId,
|
||||
kind: 'diff',
|
||||
title: 'Git diff',
|
||||
content: truncate(diff.output, 200_000),
|
||||
contentType: 'text/x-diff',
|
||||
});
|
||||
await updateStatus(jobId, 'checks_running');
|
||||
const detected = await detectPackageCommands(repoDir);
|
||||
const settings = claim.agentSettings;
|
||||
const installCommand = settings?.installCommand ?? detected.install;
|
||||
const checkCommand = settings?.checkCommand ?? detected.check;
|
||||
const testCommand = settings?.testCommand ?? detected.test;
|
||||
if (installCommand) {
|
||||
await runProjectCommand({
|
||||
command: installCommand,
|
||||
phase: 'install',
|
||||
claim,
|
||||
workdir,
|
||||
repoDir,
|
||||
redact,
|
||||
});
|
||||
}
|
||||
if (checkCommand) {
|
||||
await runProjectCommand({
|
||||
command: checkCommand,
|
||||
phase: 'check',
|
||||
claim,
|
||||
workdir,
|
||||
repoDir,
|
||||
redact,
|
||||
});
|
||||
}
|
||||
if (testCommand) {
|
||||
await runProjectCommand({
|
||||
command: testCommand,
|
||||
phase: 'test',
|
||||
claim,
|
||||
workdir,
|
||||
repoDir,
|
||||
redact,
|
||||
});
|
||||
}
|
||||
await appendEvent(jobId, 'info', 'commit', 'Committing changes.');
|
||||
const commitSha = await commitAndPush({
|
||||
repoDir,
|
||||
workBranch: claim.job.workBranch,
|
||||
message: `Agent: ${claim.job.prompt.slice(0, 72)}`,
|
||||
redact,
|
||||
timeoutMs: env.jobTimeoutMs,
|
||||
});
|
||||
const prBody = buildPrBody({
|
||||
prompt: claim.job.prompt,
|
||||
summary: edit.summary,
|
||||
commands: [
|
||||
installCommand,
|
||||
checkCommand,
|
||||
testCommand,
|
||||
...edit.commands,
|
||||
].filter((command): command is string => Boolean(command)),
|
||||
limitations: edit.limitations,
|
||||
});
|
||||
await addArtifact({
|
||||
jobId,
|
||||
kind: 'pr_body',
|
||||
title: 'Draft PR body',
|
||||
content: prBody,
|
||||
contentType: 'text/markdown',
|
||||
});
|
||||
await appendEvent(jobId, 'info', 'pr', 'Opening draft pull request.');
|
||||
const pullRequest = await openDraftPullRequest({
|
||||
installationId: claim.github.installationId,
|
||||
forkOwner: claim.job.forkOwner,
|
||||
forkRepo: claim.job.forkRepo,
|
||||
baseBranch: claim.job.baseBranch,
|
||||
workBranch: claim.job.workBranch,
|
||||
title: `Agent: ${claim.job.prompt.slice(0, 64)}`,
|
||||
body: prBody,
|
||||
});
|
||||
await completeWithDraftPr({
|
||||
jobId,
|
||||
commitSha,
|
||||
pullRequestUrl: pullRequest.html_url,
|
||||
pullRequestNumber: pullRequest.number,
|
||||
summary: edit.summary,
|
||||
});
|
||||
await appendEvent(jobId, 'info', 'cleanup', 'Agent job completed.');
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
await appendEvent(
|
||||
jobId,
|
||||
'error',
|
||||
'cleanup',
|
||||
truncate(redact(message), 20_000),
|
||||
);
|
||||
await addArtifact({
|
||||
jobId,
|
||||
kind: 'error',
|
||||
title: 'Failure',
|
||||
content: truncate(redact(message), 50_000),
|
||||
contentType: 'text/plain',
|
||||
});
|
||||
await updateStatus(
|
||||
jobId,
|
||||
message.toLowerCase().includes('timed out') ? 'timed_out' : 'failed',
|
||||
{ error: truncate(redact(message), 10_000) },
|
||||
);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
|
||||
export const startWorker = async () => {
|
||||
console.log(`Spoon agent worker ${env.workerId} polling ${env.convexUrl}`);
|
||||
for (;;) {
|
||||
try {
|
||||
const claim = await client.action(api.agentJobsNode.claimNextForWorker, {
|
||||
workerId: env.workerId,
|
||||
workerToken: env.workerToken,
|
||||
});
|
||||
if (!claim) {
|
||||
await sleep(env.pollMs);
|
||||
continue;
|
||||
}
|
||||
await runClaim(claim);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
await sleep(env.pollMs);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "@spoon/tsconfig/base.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src", "eslint.config.ts", "vitest.config.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
import { nodeProject } from '@spoon/vitest-config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
projects: [
|
||||
nodeProject('unit', ['tests/unit/**/*.test.ts']),
|
||||
nodeProject('integration', ['tests/integration/**/*.test.ts']),
|
||||
nodeProject('component', ['tests/component/**/*.test.ts']),
|
||||
],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user