docs: rebalance prose cleanup and add trimming skill
This commit is contained in:
@@ -39,4 +39,4 @@ A scenario booting a differently-composed tree sets its own `configPath` (an ove
|
||||
|
||||
Examples use a `cordis.snapshot.yml` overlay with [`dsh-llm-replay`](../llm-replay/README.md). Recording calls the live model and updates model fixtures; keyless refresh replays those fixtures and updates derived stdout, session-log, and prompt snapshots. See the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script can queue permission answers by stable option kind and can set session config options or assert their rejection. Missing permission answers cancel; selecting an unavailable kind fails the scenario.
|
||||
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script can queue permission answers by stable option kind and can set session config options or assert their rejection in the transcript. Missing permission answers cancel; selecting an unavailable kind fails the scenario.
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
/**
|
||||
* Shared subprocess harness for ACP snapshot suites. A library module driven by the suite
|
||||
* factory in ./suite.ts (and directly by harness-level specs); each example's `*.snapshot.ts`
|
||||
* names its own agent-under-test paths.
|
||||
* Shared ACP snapshot subprocess harness. It boots the real agent bin through the Cordis
|
||||
* loader, drives deterministic ACP JSON-RPC over stdio, captures protocol-pure stdout, and
|
||||
* harvests persisted session logs after graceful shutdown. Normalization stays in
|
||||
* `normalize.ts`; suite registration stays in `suite.ts`.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/harness
|
||||
*/
|
||||
|
||||
@@ -57,8 +58,8 @@ export interface AgentUnderTest {
|
||||
/**
|
||||
* One step of a scenario's deterministic input script (`input.json`). The harness interprets
|
||||
* these in order. `newSession` captures the server-issued (random) session id into a
|
||||
* `{{sessionId}}` variable that later steps reference, since a committed file cannot know the
|
||||
* id in advance.
|
||||
* `{{sessionId}}` variable that later steps reference. `promptAndCancel` sends without awaiting,
|
||||
* waits for the first streamed message, then cancels, making transcript order deterministic.
|
||||
*/
|
||||
export type InputStep =
|
||||
| { op: 'initialize'; terminalOutput?: boolean }
|
||||
@@ -75,8 +76,9 @@ export type InputStep =
|
||||
export interface InputScript {
|
||||
steps: InputStep[]
|
||||
/**
|
||||
* Ordered answers for the agent's `session/request_permission` round-trips, consumed FIFO —
|
||||
* the Nth request gets the Nth answer.
|
||||
* FIFO permission answers selected by stable option kind; the harness maps each kind to the
|
||||
* agent-issued option id. Exhaustion cancels, while a kind the agent did not offer fails the
|
||||
* scenario.
|
||||
*/
|
||||
permissionAnswers?: PermissionAnswer[]
|
||||
}
|
||||
@@ -202,8 +204,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', (c: string) => stderrChunks.push(c))
|
||||
|
||||
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO feed the
|
||||
// same bytes to the SDK client through a passthrough.
|
||||
// Tee the same raw bytes to the golden and SDK client. Decode once at the end so a UTF-8
|
||||
// sequence split across stream chunks cannot corrupt the transcript.
|
||||
const passthrough = new Readable({ read() {} })
|
||||
child.stdout.on('data', (buf: Buffer) => {
|
||||
rawBuffers.push(buf)
|
||||
@@ -226,8 +228,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
// Permission answers are consumed FIFO across the whole run; exhaustion
|
||||
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
|
||||
const permissionQueue = [...input.permissionAnswers ?? []]
|
||||
// A scenario bug detected inside a client callback (a scripted permission kind the agent
|
||||
// never offered).
|
||||
// A callback throw would become only an RPC error the agent could absorb. Record an
|
||||
// impossible permission choice here, answer cancelled, and fail the outer scenario.
|
||||
let scriptError: Error | undefined
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(params: SessionNotification): Promise<void> {
|
||||
@@ -353,7 +355,8 @@ async function runStep(
|
||||
case 'promptAndCancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
|
||||
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on its own).
|
||||
// A hang fixture never resolves alone. Wait for its streamed chunk before cancellation
|
||||
// so updates deterministically precede the cancelled prompt response.
|
||||
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
|
||||
await client.cancel({ sessionId })
|
||||
@@ -439,7 +442,8 @@ async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
|
||||
})
|
||||
}
|
||||
}
|
||||
// Primary (no parentSession) first, then children by ascending createdAt.
|
||||
// Match replay fixture assignment: primary first, then children by creation time, with id as
|
||||
// a deterministic collision tiebreaker.
|
||||
logs.sort((a, b) => {
|
||||
const ap = a.parentSession === undefined ? 0 : 1
|
||||
const bp = b.parentSession === undefined ? 0 : 1
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot tier (`pnpm run
|
||||
* test:snapshot`).
|
||||
* ACP snapshot suite kit: subprocess scenario harness, pure golden normalizers, and the Vitest
|
||||
* suite factory behind `pnpm run test:snapshot`. Because this entry exports `suite.ts`, importing
|
||||
* it requires a Vitest run.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
/**
|
||||
* Pure normalizers for the ACP snapshot goldens. They replace the non-deterministic values in
|
||||
* the two captured surfaces — the stdout JSON-RPC transcript and the persisted session JSONL —
|
||||
* with stable tokens, so a golden compare reflects behavior, not run-to-run noise. Kept
|
||||
* dependency-free and side-effect-free so they unit-test trivially.
|
||||
* Pure ACP transcript and session-log normalizers. They scrub session ids, temp cwd, RPC ids,
|
||||
* timestamps, and hook duration while preserving deterministic event sequence numbers.
|
||||
* Request-header scrubbers stay separate so one scenario per header class can pin tools and a
|
||||
* readable prompt while other fixtures omit duplicated header bulk.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/normalize
|
||||
*/
|
||||
|
||||
@@ -50,6 +50,7 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable golden
|
||||
* in the same shape as the wire: one compact JSON frame per line (NDJSON), with the JSON-RPC
|
||||
* `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all volatile strings scrubbed.
|
||||
* Invalid JSON throws, doubling as a protocol-stdout purity check.
|
||||
*
|
||||
* @param rawStdout The captured stdout bytes, decoded utf8.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
/**
|
||||
* The ACP snapshot suite factory (replay by default, keyless).
|
||||
* Keyless-by-default ACP snapshot suite factory. Each scenario drives the real subprocess and
|
||||
* compares normalized stdout; comparable session fixtures are both replay input and expected
|
||||
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
|
||||
* mode replays committed scripts and rewrites derived artifacts without a key.
|
||||
*
|
||||
* Exactly one scenario per header-composition class pins tool schemas in JSONL and the system
|
||||
* prompt in Markdown. Every live header is checked against that pin, so session-dependent
|
||||
* composition must declare a separate class instead of escaping coverage.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/suite
|
||||
*/
|
||||
|
||||
@@ -61,7 +68,8 @@ export interface Scenario {
|
||||
*/
|
||||
childSessions?: number
|
||||
/**
|
||||
* Whether this scenario pins its header class's model-facing request-header content.
|
||||
* Whether this scenario is its header class's sole request-header pin. Its Markdown file owns
|
||||
* the prompt, its JSONL keeps tool schemas, and every classmate is checked for equality.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
@@ -123,8 +131,9 @@ export function childFixturePaths(dir: string, childSessions: number): string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own header line
|
||||
* (`{ type: 'session', id, cwd }`).
|
||||
* Derive normalization values from a fixture's own session header. Recorded ids and cwd differ
|
||||
* from the live replay run; the non-empty sentinel for missing cwd avoids accidental empty-
|
||||
* string replacement.
|
||||
*
|
||||
* @param fixture The committed `session.jsonl` content.
|
||||
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
|
||||
@@ -403,8 +412,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode (recorded model scenarios only): persist the freshly-harvested live logs
|
||||
// back to their fixtures.
|
||||
// Record writes live model fixtures; keyless refresh writes every comparable replayed
|
||||
// fixture. Pins keep tools but all JSONL files scrub prompt text.
|
||||
const scrub = scenario.pinsHeader === true
|
||||
? scrubSystemPrompts
|
||||
: scrubRequestHeaders
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
/**
|
||||
* Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs.
|
||||
* Scripted ACP agent for snapshot-kit tests. A fixture-adjacent `behavior.json` controls the
|
||||
* subprocess reached through the real harness path; the bin reports observations over ACP and
|
||||
* writes scripted logs before exiting on stdin EOF.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
|
||||
@@ -308,7 +308,8 @@ describe('runScenario', () => {
|
||||
|
||||
it('rejects the run on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ permissionProbe: true })
|
||||
// The fake bin offers allow_once/reject_once; scripting allow_always is a scenario bug.
|
||||
// The fake offers only allow_once/reject_once. The harness must reject an impossible click,
|
||||
// not merely send an RPC error that a tolerant agent could absorb.
|
||||
await expect(runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
|
||||
@@ -23,6 +23,9 @@ import {
|
||||
* so every factory path — golden and log compares, the per-suite header pin and its uniformity
|
||||
* guard, record-mode fixture write-back, skip semantics, and the fixture guard block —
|
||||
* executes as an ordinary green test.
|
||||
*
|
||||
* Record tests use a temp copy. To intentionally rebuild their committed fixtures, run this
|
||||
* spec once with `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1`, then review and commit the resulting tree.
|
||||
*/
|
||||
|
||||
const AGENT = {
|
||||
|
||||
Reference in New Issue
Block a user