Merge remote-tracking branch 'origin/master' into feat/plan-mode

# Conflicts:
#	docs/config-catalog.md
#	docs/cordis-catalog/services.md
#	docs/event-producer-consumer.md
#	docs/module-graph.md
#	packages/README.md
#	packages/bash/tool-bash/src/index.ts
#	packages/core/tools/tests/gen-tool-catalog.spec.ts
#	pnpm-lock.yaml
#	scripts/doc-budgets.manifest.json
This commit is contained in:
Tianyi Cui
2026-07-17 21:22:16 +08:00
357 changed files with 18449 additions and 2870 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie
Three layers, importable separately:
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo).
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo). Startup failures preserve captured agent stderr in the rejected diagnostic.
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.golden.md` plus `tool-schemas.golden.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Must be called at vitest collection time.
@@ -192,6 +192,9 @@ export interface RunOptions {
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
// Fixed path length: spill-policy budgets the preview against the REAL path
// before stdout normalization, so tmpdir() length differences churn goldens.
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
// Everything past the temp-dir creation runs under a try/finally that always
// removes both dirs — so a failure in workspace seeding, spawn, or any step
// never leaks them (the "e2e tests own their resources" rule).
@@ -211,6 +214,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
DSH_SNAPSHOT: opts.mode,
DSH_SNAPSHOT_FILE: opts.fixtureFile,
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
DSH_SNAPSHOT_SPILL_ROOT: spillRoot,
DSH_HOME: join(cwd, '.dsh'),
DSH_AGENTS_HOME: join(cwd, '.agents'),
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
@@ -318,6 +322,10 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// Harvest EVERY persisted log (parent + any subagent children) while the
// temp dirs still exist, ordered primary-first.
sessionLogs = await harvestSessionLogs(sessionsRoot)
} catch (error: unknown) {
const stderr = stderrChunks.join('')
if (stderr === '') throw error
throw new Error(`snapshot-harness: scenario failed: ${String(error)}\nagent stderr:\n${stderr}`, { cause: error })
} finally {
// Failure-safe teardown: kill a still-running child and drop the temp dirs
// even if seeding/spawn/a step/harvest threw, so a flaky run never leaks a
@@ -328,6 +336,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
}
await rm(cwd, { recursive: true, force: true })
await rm(sessionsRoot, { recursive: true, force: true })
await rm(spillRoot, { recursive: true, force: true })
}
return {
@@ -14,6 +14,16 @@ const MESSAGE_PREFIX = '{{messagePrefix}}'
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
const LOCAL_SPILL_PATH_RE = new RegExp(
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
/** Inputs the normalizers need to recognize a run's volatile values. */
export interface NormalizeContext {
@@ -29,6 +39,9 @@ function scrubString(value: string, ctx: NormalizeContext): string {
// cwd first (longest, most specific), then explicit session ids, then any
// residual UUID (covers ids that appear in places we didn't enumerate).
out = out.split(ctx.cwd).join(CWD)
out = out.split(`/private${CWD}`).join(CWD)
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
out = out.replace(UUID_RE, SESSION_ID)
return out
+9 -6
View File
@@ -3,6 +3,8 @@
* compares normalized stdout; comparable session fixtures are both replay input and expected
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
* mode replays committed scripts and rewrites derived artifacts without a key.
* Replay scenarios run concurrently because each subprocess owns unique temp cwd and persistence
* roots and only reads committed fixtures. Record and refresh scenarios stay serial while writing.
*
* Exactly one scenario per header-composition class pins the system prompt and tool schemas in
* dedicated sidecars. Every live header is checked against that pin, so session-dependent
@@ -461,7 +463,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
}
/**
* Register the suite: one `describe` per scenario (the golden/log compares and
* Register the suite: one test per scenario (the golden/log compares and
* the header-uniformity guard) plus the fixture guard block (no orphan
* scenario dirs, required files present, exactly one pin per header class,
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
@@ -477,6 +479,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
const RECORDING = mode === 'record'
const REFRESHING = mode === 'refresh'
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
const scenarioSuite = mode === 'replay' ? describe.concurrent : describe
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
@@ -496,11 +499,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
for (const scenario of scenarios) {
describe(`snapshot: ${scenario.name}`, () => {
scenarioSuite('snapshot scenarios', () => {
for (const scenario of scenarios) {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json')
@@ -658,8 +661,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
})
})
}
}
})
describe('snapshot fixtures', () => {
it('every scenario directory is registered (no orphans)', async () => {
@@ -36,6 +36,8 @@ interface ScriptedLog {
/** The whole scripted behavior for one run. Every field defaults to the least surprising choice. */
interface Behavior {
/** Exit during startup after writing any configured stderr note. */
failOnBoot?: boolean
/** Reject every `session/new` (exercises the expect-error step without extra dirs). */
rejectNewSession?: boolean
/** Reject `session/new` only when `additionalDirectories` is non-empty (the real bridge's rule). */
@@ -78,6 +80,7 @@ const behavior: Behavior = fixtureFile === ''
: JSON.parse(readFileSync(join(dirname(fixtureFile), 'behavior.json'), 'utf8')) as Behavior
if (behavior.stderrNote !== undefined) process.stderr.write(`${behavior.stderrNote}\n`)
if (behavior.failOnBoot === true) process.exit(7)
let nextOutboundId = 1000
let sessionId = ''
@@ -38,6 +38,14 @@ async function scenario(behavior: object): Promise<{ dir: string; fixtureFile: s
const boot: InputStep[] = [{ op: 'initialize' }, { op: 'newSession' }]
describe('runScenario', () => {
it('includes agent stderr when the ACP connection closes during startup', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ failOnBoot: true, stderrNote: 'fake agent requested startup failure' })
await expect(runScenario(
{ steps: [{ op: 'initialize' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)).rejects.toThrow(/agent stderr:\nfake agent requested startup failure/)
})
it('drives a full turn: initialize (terminal caps), session, prompt, permission stub, harvest', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({
permissionProbe: true,
@@ -93,6 +93,52 @@ describe('normalizeSessionLog', () => {
expect(out).not.toContain(ctx.cwd)
})
it('scrubs random local spill paths under the snapshot cwd', () => {
const ev = JSON.stringify({
type: 'tool/result', seq: 2, time: 5,
data: {
content: [{
type: 'text',
text: `Full formatted result stored at: ${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`,
}],
},
})
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
expect(out).toContain('{{spillLocator:bash.txt}}')
expect(out).not.toContain('session-c22bc3f1d2af')
expect(out).not.toContain('8a7b6c5d4e3f')
})
it('scrubs macOS /private aliases for local spill paths', () => {
const ev = JSON.stringify({
type: 'tool/result', seq: 2, time: 5,
data: {
content: [{
type: 'text',
text: `Full formatted result stored at: /private${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`,
}],
},
})
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
expect(out).toContain('{{spillLocator:bash.txt}}')
expect(out).not.toContain('/private{{spillLocator')
})
it('scrubs fixed snapshot spill paths', () => {
const ev = JSON.stringify({
type: 'tool/result', seq: 2, time: 5,
data: {
content: [{
type: 'text',
text: 'Full formatted result stored at: /tmp/dsh-acp-snapshot-spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.',
}],
},
})
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
expect(out).toContain('{{spillLocator:bash.txt}}')
expect(out).not.toContain('/tmp/dsh-acp-snapshot-spill')
})
it('scrubs the session id in the header', () => {
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
expect(out).toContain('{{sessionId}}')