Add examples/coding-agent and the docs cookbook
The first real agent wiring: DeepSeek V4 + the bash tool suite + stdio chat + JSONL persistence, runnable via yarn demo:coding (reads the gitignored repo-root .env through process.loadEnvFile). - examples/coding-agent: cordis.yml wiring both real plugin families (llm-deepseek with !!js env secrets; bash-local + tool-bash), a bash-only coding system prompt, a max-steps-guard plugin (bounds runaway turns via the agent/turn-continuation waterfall — abort() from step-end is a no-op by then), and a stdio UI with dimmed reasoning and exit-on-idle for piped stdin. - e2e (yarn test:e2e, key-gated): full-loop.e2e.ts runs a real model against the real bash tool; coding-task.e2e.ts is the swebench-style smoke — the model fixes a buggy add.js in a temp dir and the test re-runs node add.test.js itself rather than trusting the agent. - docs/cookbook: adding-a-package (the verified checklist), adding-a-tool (execute() contract, background pattern, seams), adding-an-llm-adapter (protocol obligations, mock-server testing, e2e policy). AGENTS.md layout/commands/secrets sections updated; architecture.md points at both examples and the cookbook. - vitest.e2e.config.ts: serialize test files + retry twice — parallel e2e files trip the shared internal key's concurrency quota. - fix: the !js YAML tag spelling in docs/JSDoc is actually !!js (js-yaml resolves custom tags under tag:yaml.org,2002:js).
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { Context } from 'cordis'
|
||||
import { codingHarness, finalText, SYSTEM_PROMPT, waitForIdle } from './harness.ts'
|
||||
|
||||
/**
|
||||
* The swebench-style smoke test: a real model fixes a real bug in a temp
|
||||
* directory using only the bash tool, and the fix is verified OUTSIDE the
|
||||
* agent by re-running the test script. Key-gated.
|
||||
*/
|
||||
|
||||
const TEST_FILE = [
|
||||
"const assert = require('node:assert');",
|
||||
"const { add } = require('./add.js');",
|
||||
'assert.strictEqual(add(2, 3), 5);',
|
||||
'assert.strictEqual(add(-1, 1), 0);',
|
||||
"console.log('PASS');",
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
const BUGGY_ADD = [
|
||||
'// A tiny module with an obvious bug.',
|
||||
'function add(a, b) {',
|
||||
' return a - b;',
|
||||
'}',
|
||||
'module.exports = { add };',
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
let workdir: string | undefined
|
||||
let ctx: Context | undefined
|
||||
|
||||
afterEach(async () => {
|
||||
// Dispose the harness even on failure/retry: agent-loop teardown stops the
|
||||
// loop and LocalBashExecutor teardown kills anything the model left running.
|
||||
await ctx?.fiber.dispose()
|
||||
ctx = undefined
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
workdir = undefined
|
||||
})
|
||||
|
||||
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('coding task: fix a failing test via bash', () => {
|
||||
it('repairs add.js so node add.test.js passes', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'dsh-coding-task-'))
|
||||
await writeFile(join(workdir, 'add.js'), BUGGY_ADD)
|
||||
await writeFile(join(workdir, 'add.test.js'), TEST_FILE)
|
||||
|
||||
// Confirm the fixture actually fails before the agent touches it.
|
||||
const before = spawnSync('node', ['add.test.js'], { cwd: workdir })
|
||||
expect(before.status).not.toBe(0)
|
||||
|
||||
ctx = await codingHarness(workdir)
|
||||
const agent = ctx.agentLoop.create('e2e-task', {
|
||||
model: 'deepseek-v4-flash',
|
||||
systemPrompt: SYSTEM_PROMPT,
|
||||
})
|
||||
|
||||
agent.send([{
|
||||
type: 'text',
|
||||
text: 'In the current directory, `node add.test.js` fails because add.js has a bug. '
|
||||
+ 'Fix add.js so the test passes, run `node add.test.js` to verify, and report the result. '
|
||||
+ 'Do not modify add.test.js.',
|
||||
}])
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
// The agent claims success…
|
||||
const summary = finalText([...agent.session.events]).toLowerCase()
|
||||
expect(summary.length).toBeGreaterThan(0)
|
||||
|
||||
// …and the world agrees: the test passes when WE run it, and the test
|
||||
// file is byte-identical (an agent that neutered the test instead of
|
||||
// fixing the bug fails here, not just on a keyword probe).
|
||||
const untouchedTest = await readFile(join(workdir, 'add.test.js'), 'utf8')
|
||||
expect(untouchedTest).toBe(TEST_FILE)
|
||||
|
||||
const after = spawnSync('node', ['add.test.js'], { cwd: workdir, encoding: 'utf8' })
|
||||
expect(after.stdout).toContain('PASS')
|
||||
expect(after.status).toBe(0)
|
||||
|
||||
const fixed = await readFile(join(workdir, 'add.js'), 'utf8')
|
||||
expect(fixed).not.toMatch(/a\s*-\s*b/)
|
||||
}, 180_000)
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { Context } from 'cordis'
|
||||
import { codingHarness, finalText, SYSTEM_PROMPT, waitForIdle } from './harness.ts'
|
||||
|
||||
/**
|
||||
* The first place a REAL model meets the REAL bash tool: the cheap canary
|
||||
* before the coding-task e2e. Key-gated (see vitest.e2e.config.ts).
|
||||
*/
|
||||
|
||||
let ctx: Context | undefined
|
||||
|
||||
afterEach(async () => {
|
||||
// Always dispose the harness, even on failure/retry/timeout: agent-loop
|
||||
// teardown stops the loop and LocalBashExecutor teardown kills any
|
||||
// process the model left behind.
|
||||
await ctx?.fiber.dispose()
|
||||
ctx = undefined
|
||||
})
|
||||
|
||||
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('full loop: real model + real bash tool', () => {
|
||||
it('runs a bash command on request and reports its output', async () => {
|
||||
ctx = await codingHarness(process.cwd())
|
||||
const agent = ctx.agentLoop.create('e2e-loop', {
|
||||
model: 'deepseek-v4-flash',
|
||||
systemPrompt: SYSTEM_PROMPT,
|
||||
})
|
||||
|
||||
agent.send([{ type: 'text', text: 'Run `echo e2e-ok` with the bash tool and tell me its exact output.' }])
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
const events = [...agent.session.events]
|
||||
const calls = events.filter(event => event.type === 'tool/call')
|
||||
expect(calls.length).toBeGreaterThan(0)
|
||||
expect(calls.some(event => event.data.name === 'bash')).toBe(true)
|
||||
|
||||
const results = events.filter(event => event.type === 'tool/result')
|
||||
const resultTexts = results.flatMap(event =>
|
||||
event.data.content.filter(block => block.type === 'text').map(block => block.text))
|
||||
expect(resultTexts.some(text => text.includes('e2e-ok'))).toBe(true)
|
||||
|
||||
expect(finalText(events)).toContain('e2e-ok')
|
||||
}, 120_000)
|
||||
})
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Context } from 'cordis'
|
||||
import LlmService from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry from '@deepseek-ai/dsh-agent'
|
||||
import AgentLoop, { LoopAgent } from '@deepseek-ai/dsh-agent-loop'
|
||||
import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local'
|
||||
import * as ToolBash from '@deepseek-ai/dsh-tool-bash'
|
||||
import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
|
||||
|
||||
/**
|
||||
* Shared harness for the coding-agent e2e suites: the full plugin stack
|
||||
* with the real DeepSeek adapter and the real bash tool. Lives outside the
|
||||
* *.e2e.ts pattern so importing it never re-registers another file's tests.
|
||||
*/
|
||||
|
||||
export const SYSTEM_PROMPT = 'You are a coding agent. Your only tool is bash; '
|
||||
+ 'do file operations with cat/grep/heredocs, check [exit code: N] markers, '
|
||||
+ 'and report results briefly.'
|
||||
|
||||
export async function codingHarness(workdir: string): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(SystemPrompt)
|
||||
await ctx.plugin(ToolRegistry)
|
||||
await ctx.plugin(AgentRegistry)
|
||||
await ctx.plugin(AgentLoop, { agents: [] })
|
||||
await ctx.plugin(LlmDeepSeek, { models: ['deepseek-v4-flash'] })
|
||||
await ctx.plugin(LocalBashExecutor, { cwd: workdir, timeoutMs: 30_000 })
|
||||
await ctx.plugin(ToolBash)
|
||||
return ctx
|
||||
}
|
||||
|
||||
export function waitForIdle(ctx: Context, agent: LoopAgent): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const dispose = ctx.on('agent/status', (subject, status) => {
|
||||
if (subject === agent && status === 'idle') {
|
||||
dispose()
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export function finalText(events: SessionEvent[]): string {
|
||||
const message = events.findLast(event => event.type === 'assistant/message')
|
||||
if (message?.type !== 'assistant/message') return ''
|
||||
return message.data.content
|
||||
.filter(block => block.type === 'text')
|
||||
.map(block => block.text)
|
||||
.join('')
|
||||
}
|
||||
Reference in New Issue
Block a user