docs: trim generated prose

This commit is contained in:
Tianyi Cui
2026-07-12 03:36:43 +08:00
parent 3dca90261c
commit 75838e10b5
323 changed files with 2857 additions and 11833 deletions
+19 -70
View File
@@ -1,18 +1,7 @@
/**
* Shared subprocess harness for ACP snapshot suites. A library module driven by
* the suite factory in ./suite.ts (and directly by harness-level specs); each
* example's `*.snapshot.ts` names its own agent-under-test paths.
*
* It boots the REAL agent bin subprocess via the cordis Loader (so the
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
* and — in record mode — harvests the persisted session JSONL after a graceful
* shutdown flush. The pure normalizers in ./normalize.ts turn the captured
* stdout frames and the session-log events into stable, snapshot-able text.
*
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
*
* Shared subprocess harness for ACP snapshot suites. A library module driven by the suite
* factory in ./suite.ts (and directly by harness-level specs); each example's `*.snapshot.ts`
* names its own agent-under-test paths.
* @module @deepseek-ai/dsh-acp-snapshot/harness
*/
@@ -66,16 +55,10 @@ export interface AgentUnderTest {
}
/**
* One step of a scenario's deterministic input script (`input.json`). The
* harness interprets these in order. `newSession` captures the server-issued
* (random) session id into a `{{sessionId}}` variable that later steps
* reference, since a committed file cannot know the id in advance.
*
* `promptAndCancel` sends a prompt WITHOUT awaiting its response, waits until
* the client observes the first streamed `agent_message_chunk` (so the emitted
* frames deterministically precede the cancellation), then cancels the turn —
* the only way to exercise a cancel deterministically (a plain `prompt` step
* awaits the response, which a cancel/hang scenario would block on forever).
* One step of a scenario's deterministic input script (`input.json`). The harness interprets
* these in order. `newSession` captures the server-issued (random) session id into a
* `{{sessionId}}` variable that later steps reference, since a committed file cannot know the
* id in advance.
*/
export type InputStep =
| { op: 'initialize'; terminalOutput?: boolean }
@@ -92,16 +75,8 @@ export type InputStep =
export interface InputScript {
steps: InputStep[]
/**
* Ordered answers for the agent's `session/request_permission` round-trips,
* consumed FIFO — the Nth request gets the Nth answer. Each answer selects
* by option KIND: option ids are agent-issued randoms a committed script
* cannot know, while kinds are the ACP-stable vocabulary, so the client maps
* kind → the offered `optionId` at answer time. A request beyond the queue
* (or with no queue at all) is answered `cancelled` — the stub behavior a
* scenario without approvals relies on. A scripted kind the request does
* not offer REJECTS the run: the scenario scripted an impossible click,
* and {@link runScenario} throws once the in-flight step settles (the
* agent itself just sees `cancelled`, so it cannot absorb the bug).
* Ordered answers for the agent's `session/request_permission` round-trips, consumed FIFO —
* the Nth request gets the Nth answer.
*/
permissionAnswers?: PermissionAnswer[]
}
@@ -201,8 +176,6 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
const stderrChunks: string[] = []
try {
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
// Copied into the temp cwd so the agent's bash tools see it; the goldens
// normalize the cwd, so the seeded paths stay stable across runs.
if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
await cp(opts.workspaceDir, cwd, { recursive: true })
}
@@ -229,10 +202,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
child.stderr.setEncoding('utf8')
child.stderr.on('data', (c: string) => stderrChunks.push(c))
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO
// feed the same bytes to the SDK client through a passthrough. Buffer the raw
// bytes (not per-chunk utf8 strings) and decode once at the end, so a
// multibyte sequence split across two 'data' events can't corrupt the golden.
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO feed the
// same bytes to the SDK client through a passthrough.
const passthrough = new Readable({ read() {} })
child.stdout.on('data', (buf: Buffer) => {
rawBuffers.push(buf)
@@ -255,13 +226,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// Permission answers are consumed FIFO across the whole run; exhaustion
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
const permissionQueue = [...input.permissionAnswers ?? []]
// A scenario bug detected inside a client callback (a scripted permission
// kind the agent never offered). It cannot fail the run from in there: a
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
// a tolerant agent treats that as a denial and carries on — the run (or
// worse, a record) would absorb the impossible click silently. So the
// callback answers `cancelled` (a well-defined path for the agent),
// captures the error here, and the step loop fails the run on it.
// A scenario bug detected inside a client callback (a scripted permission kind the agent
// never offered).
let scriptError: Error | undefined
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
@@ -356,10 +322,8 @@ async function runStep(
return
}
case 'newSessionExpectError': {
// The bridge rejects a session/new that widens the workspace scope
// (non-empty additionalDirectories / mcpServers — unimplemented). The SDK
// surfaces that as a rejected RPC; swallow it so the run completes and the
// error frame is captured in the transcript.
// The bridge rejects a session/new that widens the workspace scope (non-empty
// additionalDirectories / mcpServers — unimplemented).
await client.newSession({
cwd,
mcpServers: [],
@@ -379,10 +343,8 @@ async function runStep(
case 'promptExpectError': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: promptExpectError before newSession')
// The model fails this turn (a recorded provider error), so the bridge
// answers the prompt with a JSON-RPC error and the SDK rejects. That
// rejection IS the expected editor experience — swallow it so the run
// completes and the stdout transcript (the error frame) is captured.
// The model fails this turn (a recorded provider error), so the bridge answers the prompt
// with a JSON-RPC error and the SDK rejects.
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
.then(() => { throw new Error('snapshot-harness: expected the prompt to fail but it succeeded') },
() => { /* expected: the turn failed and the bridge returned an error */ })
@@ -391,13 +353,7 @@ async function runStep(
case 'promptAndCancel': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on
// its own). To pin frame order deterministically, wait until the client
// has OBSERVED the hang's streamed agent_message_chunk before cancelling —
// so those update frames always precede the cancelled prompt response in
// the transcript (without this, the late chunk and the response race).
// Then cancel and await the prompt, which the bridge settles as
// `cancelled` once the abort propagates.
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on its own).
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
await client.cancel({ sessionId })
@@ -483,14 +439,7 @@ async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
})
}
}
// Primary (no parentSession) first, then children by ascending createdAt. A
// scenario has exactly one top-level session. In the synchronous cut sibling
// children are created strictly sequentially, so their createdAt values are
// strictly ordered; the recordedId tiebreak only keeps a degenerate
// same-millisecond collision (unreachable here) deterministic. This harvest
// order must match the replay load order in dsh-llm-replay's loadSessionScripts
// so session.<n>.jsonl maps to the same child on record and replay — replay
// re-sorts childFiles by the same key, so the two stay consistent.
// Primary (no parentSession) first, then children by ascending createdAt.
logs.sort((a, b) => {
const ap = a.parentSession === undefined ? 0 : 1
const bp = b.parentSession === undefined ? 0 : 1
+2 -13
View File
@@ -1,17 +1,6 @@
/**
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot
* tier (`pnpm run test:snapshot`). Three layers, composable per example:
* the subprocess scenario harness ({@link runScenario}), the pure golden
* normalizers ({@link normalizeStdout} / {@link normalizeSessionLog} /
* {@link scrubRequestHeaders} / {@link scrubSystemPrompts}), and the suite factory
* ({@link defineAcpSnapshotSuite}) that registers a scenario table as a full
* describe/it tree. An example's `*.snapshot.ts` supplies only its
* {@link AgentUnderTest} paths, its snapshots directory, and its
* {@link Scenario} table.
*
* NOTE: ./suite.ts imports vitest, so this package is importable only inside a
* vitest run — a support-tier constraint stated in the README.
*
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot tier (`pnpm run
* test:snapshot`).
* @module @deepseek-ai/dsh-acp-snapshot
*/
+7 -31
View File
@@ -1,29 +1,8 @@
/**
* Pure normalizers for the ACP snapshot goldens. They replace the
* non-deterministic values in the two captured surfaces — the stdout JSON-RPC
* transcript and the persisted session JSONL — with stable tokens, so a golden
* compare reflects behavior, not run-to-run noise. Kept dependency-free and
* side-effect-free so they unit-test trivially.
*
* Scrubbed: `randomUUID()` session ids → `{{sessionId}}`; the temp `mkdtemp`
* cwd → `{{cwd}}` (it appears in terminal-card `_meta` and the log header);
* JSON-RPC request `id` → a stable per-transcript sequence; the log's per-event
* `time` (epoch ms) and header `createdAt` → 0; a `hook/result` event's
* `durationMs` (wall-clock hook runtime) → 0. NOT scrubbed: the log's `seq`
* (deterministic — `seq = log.length`, part of the event-log contract).
*
* Separate, composable normalizers keep bulky request-header content out of
* session fixtures. {@link scrubSystemPrompts} replaces the composed system
* prompt in EVERY fixture; {@link scrubRequestHeaders} additionally replaces
* tool schemas and the session prefix outside each suite's header-pinning
* scenario. They are deliberately NOT folded into
* {@link normalizeSessionLog}: the suite factory composes the right scrub for
* each scenario and snapshots the pin's actual prompt as Markdown (see the
* pinned-header RFC,
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
*
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
*
* Pure normalizers for the ACP snapshot goldens. They replace the non-deterministic values in
* the two captured surfaces — the stdout JSON-RPC transcript and the persisted session JSONL —
* with stable tokens, so a golden compare reflects behavior, not run-to-run noise. Kept
* dependency-free and side-effect-free so they unit-test trivially.
* @module @deepseek-ai/dsh-acp-snapshot/normalize
*/
@@ -68,12 +47,9 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
}
/**
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a
* stable golden in the SAME shape as the wire: one compact JSON frame per line
* (NDJSON), with the JSON-RPC `id` rewritten to a per-transcript sequence
* (1, 2, 3, …) and all volatile strings scrubbed. Throws if any non-empty line
* is not valid JSON — that doubles as the stdout-purity check (no logger leaked
* onto the protocol).
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable golden
* in the same shape as the wire: one compact JSON frame per line (NDJSON), with the JSON-RPC
* `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all volatile strings scrubbed.
*
* @param rawStdout The captured stdout bytes, decoded utf8.
* @param ctx The run's volatile values to scrub.
+20 -106
View File
@@ -1,33 +1,5 @@
/**
* The ACP snapshot suite factory (REPLAY by default, keyless). A suite is a
* scenario table plus a snapshots directory: each scenario under
* `<snapshotsDir>/<name>/` ships an `input.json` (the client stdin script) and
* a `session.jsonl` fixture; replay boots the real agent subprocess
* (./harness.ts), drives it, and diffs the normalized stdout transcript
* against the committed `stdout.golden.jsonl`. For model scenarios it ALSO
* checks the re-persisted session log — against the `session.jsonl` fixture
* itself, not a separate golden: the fixture doubles as the replay source
* (recorded scenarios) and the expected produced log (both sides normalized
* before comparing).
*
* Request-header content is pinned by exactly ONE scenario per HEADER CLASS —
* scenarios that boot the same config compose the same header. Every JSONL
* fixture scrubs the system prompt to `{{system}}`; each class's pinning
* scenario stores the readable prompt in `system-prompt.golden.md` and keeps its full
* tool schemas in `session.jsonl`, while every other fixture also scrubs tools
* to `{{tools}}`. A per-run uniformity guard compares both artifacts against
* every live header and forbids unrepresented header deltas (see the
* pinned-header RFC,
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
*
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
* in one pass. `pnpm run test:snapshot:refresh` (DSH_SNAPSHOT=refresh) instead
* replays the committed model scripts keylessly and writes the current stdout
* + persisted-log goldens back without calling a live LLM. The caller resolves
* that env into {@link SnapshotSuiteOptions} (env reading stays at the suite
* edge, not in this library).
*
* The ACP snapshot suite factory (replay by default, keyless).
* @module @deepseek-ai/dsh-acp-snapshot/suite
*/
@@ -89,20 +61,7 @@ export interface Scenario {
*/
childSessions?: number
/**
* Whether THIS scenario pins its header class's model-facing request-header
* content. Its actual composed prompt is maintained as a readable
* `system-prompt.golden.md`; its JSONL keeps full tool schemas but stores the prompt
* as `{{system}}`. Every other scenario of the class stores tools as
* `{{tools}}` too ({@link scrubRequestHeaders}). A prompt or tool-schema
* change therefore shows up in one focused artifact per class, not every
* session fixture. One pin per class suffices because
* header composition is class-uniform (parent, spawn child, and fork child
* all compose the same prompt-modulo-cwd and the same tools) — and that
* premise is ASSERTED, not assumed: every non-pinning run's live headers
* must equal its class's pinned fixture's (normalized), so a
* session-dependent header (say, a restricted subagent toolset) fails loud
* until it gets its own pinning scenario.
* Defaults to false.
* Whether this scenario pins its header class's model-facing request-header content.
*/
pinsHeader?: boolean
/**
@@ -164,17 +123,8 @@ export function childFixturePaths(dir: string, childSessions: number): string[]
}
/**
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own
* header line (`{ type: 'session', id, cwd }`). A committed fixture carries the
* session id and cwd of the run that harvested it — different from the live
* replay run — so normalizing it against the live run's ctx would leave those
* recorded values unscrubbed. Reading them from the header scrubs the fixture's
* own id/cwd to the same `{{sessionId}}`/`{{cwd}}` tokens the replay output gets.
* An authored fixture whose header is already normalized (`id:'{{sessionId}}'`,
* `cwd:'{{cwd}}'`) yields those tokens as the volatile values, so scrubbing them
* is an idempotent no-op. A header with no `cwd` falls back to a sentinel that
* cannot occur in a log (NOT `''`, which `String.split` would match on every
* character boundary and corrupt the output).
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own header line
* (`{ type: 'session', id, cwd }`).
*
* @param fixture The committed `session.jsonl` content.
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
@@ -419,10 +369,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
for (const scenario of scenarios) {
describe(`snapshot: ${scenario.name}`, () => {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
// REFRESH mode is replay-backed and deterministic, so it runs every
// scenario and rewrites the comparable fixtures from that replay run.
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
@@ -444,10 +392,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
...scenario.configPath !== undefined ? { configPath: scenario.configPath } : {},
})
// Scrub every volatile id the run produced: the ACP server-issued session
// id plus every harvested log's recorded id (a subagent child id never
// surfaces over ACP, but it appears in the child's own log header). The
// normalizer's UUID catch-all covers any we don't enumerate.
// Scrub every volatile id the run produced: the ACP server-issued session id plus every
// harvested log's recorded id (a subagent child id never surfaces over ACP, but it
// appears in the child's own log header).
const ctx: NormalizeContext = {
sessionIds: [
...result.sessionId !== undefined ? [result.sessionId] : [],
@@ -456,15 +403,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
cwd: result.cwd,
}
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
// live logs back to their fixtures. REFRESH mode does the same from a
// keyless replay run for every comparable log, including authored
// scenarios that live record deliberately skips. The primary goes to
// session.jsonl, each child to session.<n>.jsonl in harvest order. A
// Every fixture is written with its system prompt scrubbed. A pinning
// scenario keeps the remaining header content (notably tool schemas);
// every other scenario scrubs that bulk too. Record/refresh therefore
// cannot smuggle prompt text back into JSONL or duplicate schemas.
// RECORD mode (recorded model scenarios only): persist the freshly-harvested live logs
// back to their fixtures.
const scrub = scenario.pinsHeader === true
? scrubSystemPrompts
: scrubRequestHeaders
@@ -515,14 +455,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
// A model turn always produces a log worth comparing; a hook scenario can
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
if (comparesLog) {
// The harvested logs (primary-first) must match their committed fixtures
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
// OWN volatile values — the live run's via `ctx`, the committed fixture's
// via its own header (a committed file cannot share the live run's ids).
// Both sides pass through the scenario's idempotent scrub: every live
// prompt becomes the fixture's `{{system}}`; non-pinning scenarios
// additionally tokenize tools/prefix. The dedicated header guard below
// compares those omitted values against their class's pin artifacts.
// The harvested logs (primary-first) must match their committed fixtures 1:1.
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
for (let i = 0; i < fixtureFiles.length; i++) {
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
@@ -532,11 +465,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
// Header-uniformity guard: every live header in a class must equal the
// class pin split across its JSONL header (system token + real tools)
// and readable Markdown prompt. A pinning scenario may carry its
// declared header deltas; their prompt edits live in the Markdown
// golden while JSONL retains the tokenized edit structure.
// Header-uniformity guard: every live header in a class must equal the class pin split
// across its JSONL header (system token + real tools) and readable Markdown prompt.
/* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */
const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario
const pinningDir = join(snapshotsDir, pinningScenario.name)
@@ -586,18 +516,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
})
it('every registered scenario has its required fixture files', () => {
// Every scenario has an input script and an stdout golden. EVERY scenario
// also needs `session.jsonl`: the suite boots `llm-replay` with that path
// as the replay source for ALL scenarios (the factory passes
// `fixtureFile: <dir>/session.jsonl` unconditionally), and `loadReplayScript`
// throws "fixture not found" when it is absent and no override replaces it.
// A no-model scenario ships a header-only `session.jsonl` (it derives to an
// empty script — no model call is made); a model scenario's fixture also
// doubles as the expected-log artifact the run is diffed against. The
// `replay.override.json` sidecar is matched BOTH ways against the table's
// `overridden` flag: required when set, forbidden when not — the harness
// forwards the file purely on existence, so an unregistered stray sidecar
// would silently replace the derived script.
// Every scenario has an input script and an stdout golden.
for (const { name, overridden, childSessions, pinsHeader } of scenarios) {
const dir = join(snapshotsDir, name)
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
@@ -616,10 +535,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
})
it('exactly one scenario pins the request-header content of each header class', () => {
// Zero pins would drop a class's prompt/schema surface from the suite
// entirely; two would split it. One pin per class is the design
// (pinned-header RFC); WHICH scenario pins is the scenario table's
// reviewable choice.
// Zero pins would drop a class's prompt/schema surface from the suite entirely; two would
// split it.
const pins = new Map<string, string[]>()
for (const scenario of scenarios.filter(s => s.pinsHeader === true)) {
const cls = classOf(scenario)
@@ -633,12 +550,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
})
it('every pinning fixture carries one request/header, one readable prompt, and its declared deltas', async () => {
// The live uniformity guard runs only in NON-pinning scenarios, so a
// class made of just its pinning scenario would otherwise accept a
// re-recorded pin with several headers or an undeclared mid-run
// header-delta — shapes the pin design cannot represent. Assert the
// committed pins directly; a scenario whose arc legitimately rewrites
// a prompt section declares the exact count via expectedHeaderDeltas.
// The live uniformity guard runs only in NON-pinning scenarios, so a class made of just
// its pinning scenario would otherwise accept a re-recorded pin with several headers or
// an undeclared mid-run header-delta — shapes the pin design cannot represent.
for (const scenario of pinningByClass.values()) {
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
const headers = normalizedHeaders(fixture, fixtureContext(fixture))