docs: rebalance prose cleanup and add trimming skill

This commit is contained in:
Tianyi Cui
2026-07-13 23:27:00 +08:00
parent fcdc318dda
commit 148046b9c8
392 changed files with 2801 additions and 1754 deletions
+1 -1
View File
@@ -39,4 +39,4 @@ A scenario booting a differently-composed tree sets its own `configPath` (an ove
Examples use a `cordis.snapshot.yml` overlay with [`dsh-llm-replay`](../llm-replay/README.md). Recording calls the live model and updates model fixtures; keyless refresh replays those fixtures and updates derived stdout, session-log, and prompt snapshots. See the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script can queue permission answers by stable option kind and can set session config options or assert their rejection. Missing permission answers cancel; selecting an unavailable kind fails the scenario.
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script can queue permission answers by stable option kind and can set session config options or assert their rejection in the transcript. Missing permission answers cancel; selecting an unavailable kind fails the scenario.
+17 -13
View File
@@ -1,7 +1,8 @@
/**
* Shared subprocess harness for ACP snapshot suites. A library module driven by the suite
* factory in ./suite.ts (and directly by harness-level specs); each example's `*.snapshot.ts`
* names its own agent-under-test paths.
* Shared ACP snapshot subprocess harness. It boots the real agent bin through the Cordis
* loader, drives deterministic ACP JSON-RPC over stdio, captures protocol-pure stdout, and
* harvests persisted session logs after graceful shutdown. Normalization stays in
* `normalize.ts`; suite registration stays in `suite.ts`.
* @module @deepseek-ai/dsh-acp-snapshot/harness
*/
@@ -57,8 +58,8 @@ export interface AgentUnderTest {
/**
* One step of a scenario's deterministic input script (`input.json`). The harness interprets
* these in order. `newSession` captures the server-issued (random) session id into a
* `{{sessionId}}` variable that later steps reference, since a committed file cannot know the
* id in advance.
* `{{sessionId}}` variable that later steps reference. `promptAndCancel` sends without awaiting,
* waits for the first streamed message, then cancels, making transcript order deterministic.
*/
export type InputStep =
| { op: 'initialize'; terminalOutput?: boolean }
@@ -75,8 +76,9 @@ export type InputStep =
export interface InputScript {
steps: InputStep[]
/**
* Ordered answers for the agent's `session/request_permission` round-trips, consumed FIFO —
* the Nth request gets the Nth answer.
* FIFO permission answers selected by stable option kind; the harness maps each kind to the
* agent-issued option id. Exhaustion cancels, while a kind the agent did not offer fails the
* scenario.
*/
permissionAnswers?: PermissionAnswer[]
}
@@ -202,8 +204,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
child.stderr.setEncoding('utf8')
child.stderr.on('data', (c: string) => stderrChunks.push(c))
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO feed the
// same bytes to the SDK client through a passthrough.
// Tee the same raw bytes to the golden and SDK client. Decode once at the end so a UTF-8
// sequence split across stream chunks cannot corrupt the transcript.
const passthrough = new Readable({ read() {} })
child.stdout.on('data', (buf: Buffer) => {
rawBuffers.push(buf)
@@ -226,8 +228,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// Permission answers are consumed FIFO across the whole run; exhaustion
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
const permissionQueue = [...input.permissionAnswers ?? []]
// A scenario bug detected inside a client callback (a scripted permission kind the agent
// never offered).
// A callback throw would become only an RPC error the agent could absorb. Record an
// impossible permission choice here, answer cancelled, and fail the outer scenario.
let scriptError: Error | undefined
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
@@ -353,7 +355,8 @@ async function runStep(
case 'promptAndCancel': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on its own).
// A hang fixture never resolves alone. Wait for its streamed chunk before cancellation
// so updates deterministically precede the cancelled prompt response.
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
await client.cancel({ sessionId })
@@ -439,7 +442,8 @@ async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
})
}
}
// Primary (no parentSession) first, then children by ascending createdAt.
// Match replay fixture assignment: primary first, then children by creation time, with id as
// a deterministic collision tiebreaker.
logs.sort((a, b) => {
const ap = a.parentSession === undefined ? 0 : 1
const bp = b.parentSession === undefined ? 0 : 1
+3 -2
View File
@@ -1,6 +1,7 @@
/**
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot tier (`pnpm run
* test:snapshot`).
* ACP snapshot suite kit: subprocess scenario harness, pure golden normalizers, and the Vitest
* suite factory behind `pnpm run test:snapshot`. Because this entry exports `suite.ts`, importing
* it requires a Vitest run.
* @module @deepseek-ai/dsh-acp-snapshot
*/
@@ -1,8 +1,8 @@
/**
* Pure normalizers for the ACP snapshot goldens. They replace the non-deterministic values in
* the two captured surfaces — the stdout JSON-RPC transcript and the persisted session JSONL —
* with stable tokens, so a golden compare reflects behavior, not run-to-run noise. Kept
* dependency-free and side-effect-free so they unit-test trivially.
* Pure ACP transcript and session-log normalizers. They scrub session ids, temp cwd, RPC ids,
* timestamps, and hook duration while preserving deterministic event sequence numbers.
* Request-header scrubbers stay separate so one scenario per header class can pin tools and a
* readable prompt while other fixtures omit duplicated header bulk.
* @module @deepseek-ai/dsh-acp-snapshot/normalize
*/
@@ -50,6 +50,7 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable golden
* in the same shape as the wire: one compact JSON frame per line (NDJSON), with the JSON-RPC
* `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all volatile strings scrubbed.
* Invalid JSON throws, doubling as a protocol-stdout purity check.
*
* @param rawStdout The captured stdout bytes, decoded utf8.
* @param ctx The run's volatile values to scrub.
+15 -6
View File
@@ -1,5 +1,12 @@
/**
* The ACP snapshot suite factory (replay by default, keyless).
* Keyless-by-default ACP snapshot suite factory. Each scenario drives the real subprocess and
* compares normalized stdout; comparable session fixtures are both replay input and expected
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
* mode replays committed scripts and rewrites derived artifacts without a key.
*
* Exactly one scenario per header-composition class pins tool schemas in JSONL and the system
* prompt in Markdown. Every live header is checked against that pin, so session-dependent
* composition must declare a separate class instead of escaping coverage.
* @module @deepseek-ai/dsh-acp-snapshot/suite
*/
@@ -61,7 +68,8 @@ export interface Scenario {
*/
childSessions?: number
/**
* Whether this scenario pins its header class's model-facing request-header content.
* Whether this scenario is its header class's sole request-header pin. Its Markdown file owns
* the prompt, its JSONL keeps tool schemas, and every classmate is checked for equality.
*/
pinsHeader?: boolean
/**
@@ -123,8 +131,9 @@ export function childFixturePaths(dir: string, childSessions: number): string[]
}
/**
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own header line
* (`{ type: 'session', id, cwd }`).
* Derive normalization values from a fixture's own session header. Recorded ids and cwd differ
* from the live replay run; the non-empty sentinel for missing cwd avoids accidental empty-
* string replacement.
*
* @param fixture The committed `session.jsonl` content.
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
@@ -403,8 +412,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
cwd: result.cwd,
}
// RECORD mode (recorded model scenarios only): persist the freshly-harvested live logs
// back to their fixtures.
// Record writes live model fixtures; keyless refresh writes every comparable replayed
// fixture. Pins keep tools but all JSONL files scrub prompt text.
const scrub = scenario.pinsHeader === true
? scrubSystemPrompts
: scrubRequestHeaders
@@ -1,5 +1,7 @@
/**
* Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs.
* Scripted ACP agent for snapshot-kit tests. A fixture-adjacent `behavior.json` controls the
* subprocess reached through the real harness path; the bin reports observations over ACP and
* writes scripted logs before exiting on stdin EOF.
*/
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
@@ -308,7 +308,8 @@ describe('runScenario', () => {
it('rejects the run on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
// The fake bin offers allow_once/reject_once; scripting allow_always is a scenario bug.
// The fake offers only allow_once/reject_once. The harness must reject an impossible click,
// not merely send an RPC error that a tolerant agent could absorb.
await expect(runScenario(
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
@@ -23,6 +23,9 @@ import {
* so every factory path — golden and log compares, the per-suite header pin and its uniformity
* guard, record-mode fixture write-back, skip semantics, and the fixture guard block —
* executes as an ordinary green test.
*
* Record tests use a temp copy. To intentionally rebuild their committed fixtures, run this
* spec once with `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1`, then review and commit the resulting tree.
*/
const AGENT = {