Merge origin/master into codex/truncated-design
This commit is contained in:
@@ -5,8 +5,8 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie
|
||||
Three layers, importable separately:
|
||||
|
||||
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo).
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}` in every JSONL), and `scrubRequestHeaders` (the remaining header bulk → `{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, the per-header-class pin (`system-prompt.golden.md` plus the JSONL's full tool schemas) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt-scrubbed, non-pinning fixtures fully header-scrubbed). Must be called at vitest collection time.
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.golden.md` plus `tool-schemas.golden.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Must be called at vitest collection time.
|
||||
|
||||
A consuming `*.snapshot.ts` is the scenario table plus one factory call:
|
||||
|
||||
@@ -21,7 +21,7 @@ const SCENARIOS: Scenario[] = [
|
||||
|
||||
defineAcpSnapshotSuite({
|
||||
agent: { // absolute paths, resolved from the suite's own location
|
||||
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
|
||||
binScript: fileURLToPath(new URL('../../../packages/examples/acp-demo/src/bin.ts', import.meta.url)),
|
||||
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
|
||||
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
|
||||
},
|
||||
@@ -35,8 +35,17 @@ defineAcpSnapshotSuite({
|
||||
})
|
||||
```
|
||||
|
||||
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode scenarios are the template. Each pinning directory's generated `system-prompt.golden.md` is the reviewable snapshot of the normalized composed prompt; `session.jsonl` stores `"system":"{{system}}"` while retaining the complete tool list.
|
||||
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode and filesystem scenarios are templates. Each pinning directory stores the normalized composed prompt in generated `system-prompt.golden.md` and the initial schemas plus schema deltas in generated `tool-schemas.golden.json`; `session.jsonl` stores `"system":"{{system}}","tools":"{{tools}}"` while retaining config, reason, and any model-visible prefix.
|
||||
|
||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log goldens, and each pin's Markdown prompt snapshot from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
Examples use a `cordis.snapshot.yml` overlay with [`dsh-llm-replay`](../llm-replay/README.md). Recording calls the live model and updates model fixtures; keyless refresh replays those fixtures and updates derived stdout, session-log, prompt, and tool-schema snapshots. See the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug). Session config options are scriptable too: the `setConfigOption` step switches a knob over `session/set_config_option`, and `setConfigOptionExpectError` asserts the bridge rejects an unknown id or out-of-vocabulary value (the error frame stays in the transcript).
|
||||
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script queues permission answers by stable option kind and maps them to current option ids; a missing answer cancels, while an unavailable kind fails the scenario after cancelling the agent request. It can also set session config options or assert that unknown ids and values are rejected in the transcript.
|
||||
|
||||
## Model Experience
|
||||
|
||||
None, as this test-only harness records, normalizes, and compares ACP transcripts without changing the agent's assembled model request.
|
||||
|
||||
## Known Limitations and Deferred Work
|
||||
|
||||
- **Session harvest is JSONL-only** — `runScenario` collects persisted `.jsonl` logs, so an example composed over the SQLite persistence backend has no snapshot path.
|
||||
- **The subprocess boots the unbuilt tsx/Loader path only** — the built-bin artifact is guarded by the separate `built-bin` e2e smokes, never by this tier.
|
||||
|
||||
@@ -27,9 +27,9 @@
|
||||
"vitest": "^4.1.8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
"cordis": "^4.0.0-rc.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
"cordis": "^4.0.0-rc.7"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,8 @@
|
||||
/**
|
||||
* Shared subprocess harness for ACP snapshot suites. A library module driven by
|
||||
* the suite factory in ./suite.ts (and directly by harness-level specs); each
|
||||
* example's `*.snapshot.ts` names its own agent-under-test paths.
|
||||
*
|
||||
* It boots the REAL agent bin subprocess via the cordis Loader (so the
|
||||
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
|
||||
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
|
||||
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
|
||||
* and — in record mode — harvests the persisted session JSONL after a graceful
|
||||
* shutdown flush. The pure normalizers in ./normalize.ts turn the captured
|
||||
* stdout frames and the session-log events into stable, snapshot-able text.
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*
|
||||
* Shared ACP snapshot subprocess harness. It boots the real agent bin through the Cordis
|
||||
* loader, drives deterministic ACP JSON-RPC over stdio, captures protocol-pure stdout, and
|
||||
* harvests persisted session logs after graceful shutdown. Normalization stays in
|
||||
* `normalize.ts`; suite registration stays in `suite.ts`.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/harness
|
||||
*/
|
||||
|
||||
@@ -47,7 +37,7 @@ const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
|
||||
* them from its own `import.meta.url`.
|
||||
*/
|
||||
export interface AgentUnderTest {
|
||||
/** The agent bin entry (e.g. `packages/ui/acp-agent/src/bin.ts`), run unbuilt via tsx. */
|
||||
/** The agent bin entry (e.g. `packages/examples/acp-demo/src/bin.ts`), run unbuilt via tsx. */
|
||||
binScript: string
|
||||
/**
|
||||
* The example's live `cordis.yml`. Under `DSH_SNAPSHOT=replay` the bin swaps
|
||||
@@ -66,16 +56,10 @@ export interface AgentUnderTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* One step of a scenario's deterministic input script (`input.json`). The
|
||||
* harness interprets these in order. `newSession` captures the server-issued
|
||||
* (random) session id into a `{{sessionId}}` variable that later steps
|
||||
* reference, since a committed file cannot know the id in advance.
|
||||
*
|
||||
* `promptAndCancel` sends a prompt WITHOUT awaiting its response, waits until
|
||||
* the client observes the first streamed `agent_message_chunk` (so the emitted
|
||||
* frames deterministically precede the cancellation), then cancels the turn —
|
||||
* the only way to exercise a cancel deterministically (a plain `prompt` step
|
||||
* awaits the response, which a cancel/hang scenario would block on forever).
|
||||
* One step of a scenario's deterministic input script (`input.json`). The harness interprets
|
||||
* these in order. `newSession` captures the server-issued (random) session id into a
|
||||
* `{{sessionId}}` variable that later steps reference. `promptAndCancel` sends without awaiting,
|
||||
* waits for the first streamed message, then cancels, making transcript order deterministic.
|
||||
*/
|
||||
export type InputStep =
|
||||
| { op: 'initialize'; terminalOutput?: boolean }
|
||||
@@ -92,16 +76,9 @@ export type InputStep =
|
||||
export interface InputScript {
|
||||
steps: InputStep[]
|
||||
/**
|
||||
* Ordered answers for the agent's `session/request_permission` round-trips,
|
||||
* consumed FIFO — the Nth request gets the Nth answer. Each answer selects
|
||||
* by option KIND: option ids are agent-issued randoms a committed script
|
||||
* cannot know, while kinds are the ACP-stable vocabulary, so the client maps
|
||||
* kind → the offered `optionId` at answer time. A request beyond the queue
|
||||
* (or with no queue at all) is answered `cancelled` — the stub behavior a
|
||||
* scenario without approvals relies on. A scripted kind the request does
|
||||
* not offer REJECTS the run: the scenario scripted an impossible click,
|
||||
* and {@link runScenario} throws once the in-flight step settles (the
|
||||
* agent itself just sees `cancelled`, so it cannot absorb the bug).
|
||||
* FIFO permission answers selected by stable option kind; the harness maps each kind to the
|
||||
* agent-issued option id. Exhaustion cancels, while a kind the agent did not offer fails the
|
||||
* scenario.
|
||||
*/
|
||||
permissionAnswers?: PermissionAnswer[]
|
||||
}
|
||||
@@ -204,8 +181,6 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
const stderrChunks: string[] = []
|
||||
try {
|
||||
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
|
||||
// Copied into the temp cwd so the agent's bash tools see it; the goldens
|
||||
// normalize the cwd, so the seeded paths stay stable across runs.
|
||||
if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
|
||||
await cp(opts.workspaceDir, cwd, { recursive: true })
|
||||
}
|
||||
@@ -233,10 +208,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', (c: string) => stderrChunks.push(c))
|
||||
|
||||
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO
|
||||
// feed the same bytes to the SDK client through a passthrough. Buffer the raw
|
||||
// bytes (not per-chunk utf8 strings) and decode once at the end, so a
|
||||
// multibyte sequence split across two 'data' events can't corrupt the golden.
|
||||
// Tee the same raw bytes to the golden and SDK client. Decode once at the end so a UTF-8
|
||||
// sequence split across stream chunks cannot corrupt the transcript.
|
||||
const passthrough = new Readable({ read() {} })
|
||||
child.stdout.on('data', (buf: Buffer) => {
|
||||
rawBuffers.push(buf)
|
||||
@@ -259,13 +232,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
// Permission answers are consumed FIFO across the whole run; exhaustion
|
||||
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
|
||||
const permissionQueue = [...input.permissionAnswers ?? []]
|
||||
// A scenario bug detected inside a client callback (a scripted permission
|
||||
// kind the agent never offered). It cannot fail the run from in there: a
|
||||
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
|
||||
// a tolerant agent treats that as a denial and carries on — the run (or
|
||||
// worse, a record) would absorb the impossible click silently. So the
|
||||
// callback answers `cancelled` (a well-defined path for the agent),
|
||||
// captures the error here, and the step loop fails the run on it.
|
||||
// A callback throw would become only an RPC error the agent could absorb. Record an
|
||||
// impossible permission choice here, answer cancelled, and fail the outer scenario.
|
||||
let scriptError: Error | undefined
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(params: SessionNotification): Promise<void> {
|
||||
@@ -361,10 +329,8 @@ async function runStep(
|
||||
return
|
||||
}
|
||||
case 'newSessionExpectError': {
|
||||
// The bridge rejects a session/new that widens the workspace scope
|
||||
// (non-empty additionalDirectories / mcpServers — unimplemented). The SDK
|
||||
// surfaces that as a rejected RPC; swallow it so the run completes and the
|
||||
// error frame is captured in the transcript.
|
||||
// The bridge rejects a session/new that widens the workspace scope (non-empty
|
||||
// additionalDirectories / mcpServers — unimplemented).
|
||||
await client.newSession({
|
||||
cwd,
|
||||
mcpServers: [],
|
||||
@@ -384,10 +350,8 @@ async function runStep(
|
||||
case 'promptExpectError': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptExpectError before newSession')
|
||||
// The model fails this turn (a recorded provider error), so the bridge
|
||||
// answers the prompt with a JSON-RPC error and the SDK rejects. That
|
||||
// rejection IS the expected editor experience — swallow it so the run
|
||||
// completes and the stdout transcript (the error frame) is captured.
|
||||
// The model fails this turn (a recorded provider error), so the bridge answers the prompt
|
||||
// with a JSON-RPC error and the SDK rejects.
|
||||
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
.then(() => { throw new Error('snapshot-harness: expected the prompt to fail but it succeeded') },
|
||||
() => { /* expected: the turn failed and the bridge returned an error */ })
|
||||
@@ -396,13 +360,8 @@ async function runStep(
|
||||
case 'promptAndCancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
|
||||
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on
|
||||
// its own). To pin frame order deterministically, wait until the client
|
||||
// has OBSERVED the hang's streamed agent_message_chunk before cancelling —
|
||||
// so those update frames always precede the cancelled prompt response in
|
||||
// the transcript (without this, the late chunk and the response race).
|
||||
// Then cancel and await the prompt, which the bridge settles as
|
||||
// `cancelled` once the abort propagates.
|
||||
// A hang fixture never resolves alone. Wait for its streamed chunk before cancellation
|
||||
// so updates deterministically precede the cancelled prompt response.
|
||||
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
|
||||
await client.cancel({ sessionId })
|
||||
@@ -488,14 +447,8 @@ async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
|
||||
})
|
||||
}
|
||||
}
|
||||
// Primary (no parentSession) first, then children by ascending createdAt. A
|
||||
// scenario has exactly one top-level session. In the synchronous cut sibling
|
||||
// children are created strictly sequentially, so their createdAt values are
|
||||
// strictly ordered; the recordedId tiebreak only keeps a degenerate
|
||||
// same-millisecond collision (unreachable here) deterministic. This harvest
|
||||
// order must match the replay load order in dsh-llm-replay's loadSessionScripts
|
||||
// so session.<n>.jsonl maps to the same child on record and replay — replay
|
||||
// re-sorts childFiles by the same key, so the two stay consistent.
|
||||
// Match replay fixture assignment: primary first, then children by creation time, with id as
|
||||
// a deterministic collision tiebreaker.
|
||||
logs.sort((a, b) => {
|
||||
const ap = a.parentSession === undefined ? 0 : 1
|
||||
const bp = b.parentSession === undefined ? 0 : 1
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
/**
|
||||
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot
|
||||
* tier (`pnpm run test:snapshot`). Three layers, composable per example:
|
||||
* the subprocess scenario harness ({@link runScenario}), the pure golden
|
||||
* normalizers ({@link normalizeStdout} / {@link normalizeSessionLog} /
|
||||
* {@link scrubRequestHeaders} / {@link scrubSystemPrompts}), and the suite factory
|
||||
* ({@link defineAcpSnapshotSuite}) that registers a scenario table as a full
|
||||
* describe/it tree. An example's `*.snapshot.ts` supplies only its
|
||||
* {@link AgentUnderTest} paths, its snapshots directory, and its
|
||||
* {@link Scenario} table.
|
||||
*
|
||||
* NOTE: ./suite.ts imports vitest, so this package is importable only inside a
|
||||
* vitest run — a support-tier constraint stated in the README.
|
||||
*
|
||||
* ACP snapshot suite kit: subprocess scenario harness, pure golden normalizers, and the Vitest
|
||||
* suite factory behind `pnpm run test:snapshot`. Because this entry exports `suite.ts`, importing
|
||||
* it requires a Vitest run.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot
|
||||
*/
|
||||
|
||||
@@ -30,6 +20,7 @@ export {
|
||||
normalizeStdout,
|
||||
scrubRequestHeaders,
|
||||
scrubSystemPrompts,
|
||||
scrubToolSchemas,
|
||||
type NormalizeContext,
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
|
||||
@@ -1,29 +1,8 @@
|
||||
/**
|
||||
* Pure normalizers for the ACP snapshot goldens. They replace the
|
||||
* non-deterministic values in the two captured surfaces — the stdout JSON-RPC
|
||||
* transcript and the persisted session JSONL — with stable tokens, so a golden
|
||||
* compare reflects behavior, not run-to-run noise. Kept dependency-free and
|
||||
* side-effect-free so they unit-test trivially.
|
||||
*
|
||||
* Scrubbed: `randomUUID()` session ids → `{{sessionId}}`; the temp `mkdtemp`
|
||||
* cwd → `{{cwd}}` (it appears in terminal-card `_meta` and the log header);
|
||||
* JSON-RPC request `id` → a stable per-transcript sequence; the log's per-event
|
||||
* `time` (epoch ms) and header `createdAt` → 0; a `hook/result` event's
|
||||
* `durationMs` (wall-clock hook runtime) → 0. NOT scrubbed: the log's `seq`
|
||||
* (deterministic — `seq = log.length`, part of the event-log contract).
|
||||
*
|
||||
* Separate, composable normalizers keep bulky request-header content out of
|
||||
* session fixtures. {@link scrubSystemPrompts} replaces the composed system
|
||||
* prompt in EVERY fixture; {@link scrubRequestHeaders} additionally replaces
|
||||
* tool schemas and the session prefix outside each suite's header-pinning
|
||||
* scenario. They are deliberately NOT folded into
|
||||
* {@link normalizeSessionLog}: the suite factory composes the right scrub for
|
||||
* each scenario and snapshots the pin's actual prompt as Markdown (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*
|
||||
* Pure ACP transcript and session-log normalizers. They scrub session ids, temp cwd, RPC ids,
|
||||
* timestamps, and hook duration while preserving deterministic event sequence numbers.
|
||||
* Request-header scrubbers stay composable so one scenario per header class can pin prompt and
|
||||
* tool-schema sidecars while retaining any model-visible prefix in the session log.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/normalize
|
||||
*/
|
||||
|
||||
@@ -81,12 +60,10 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a
|
||||
* stable golden in the SAME shape as the wire: one compact JSON frame per line
|
||||
* (NDJSON), with the JSON-RPC `id` rewritten to a per-transcript sequence
|
||||
* (1, 2, 3, …) and all volatile strings scrubbed. Throws if any non-empty line
|
||||
* is not valid JSON — that doubles as the stdout-purity check (no logger leaked
|
||||
* onto the protocol).
|
||||
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable golden
|
||||
* in the same shape as the wire: one compact JSON frame per line (NDJSON), with the JSON-RPC
|
||||
* `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all volatile strings scrubbed.
|
||||
* Invalid JSON throws, doubling as a protocol-stdout purity check.
|
||||
*
|
||||
* @param rawStdout The captured stdout bytes, decoded utf8.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
@@ -159,7 +136,21 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
||||
* @returns The JSONL with system-prompt content tokenized.
|
||||
*/
|
||||
export function scrubSystemPrompts(rawLog: string): string {
|
||||
return scrubHeaderContent(rawLog, false)
|
||||
return scrubHeaderContent(rawLog, { system: true })
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace tool schemas in request headers and header deltas with `{{tools}}`
|
||||
* tokens while retaining field presence, tool names, and delta structure.
|
||||
* System prompts and session-prefix messages stay verbatim so pinning fixtures
|
||||
* can move only schema bulk into their dedicated JSON sidecar. Lines without a
|
||||
* tool payload pass through byte-for-byte; the transform is idempotent.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @returns The JSONL with tool-schema content tokenized.
|
||||
*/
|
||||
export function scrubToolSchemas(rawLog: string): string {
|
||||
return scrubHeaderContent(rawLog, { tools: true })
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -174,11 +165,18 @@ export function scrubSystemPrompts(rawLog: string): string {
|
||||
* @returns The JSONL with all header bulk tokenized, other lines byte-identical.
|
||||
*/
|
||||
export function scrubRequestHeaders(rawLog: string): string {
|
||||
return scrubHeaderContent(rawLog, true)
|
||||
return scrubHeaderContent(rawLog, { system: true, tools: true, prefix: true })
|
||||
}
|
||||
|
||||
/** Transform header content, optionally including tool schemas and the session prefix. */
|
||||
function scrubHeaderContent(rawLog: string, scrubToolsAndPrefix: boolean): string {
|
||||
/** Which independent request-header payloads a scrubber replaces. */
|
||||
interface HeaderScrubOptions {
|
||||
system?: boolean
|
||||
tools?: boolean
|
||||
prefix?: boolean
|
||||
}
|
||||
|
||||
/** Transform the selected request-header payloads. */
|
||||
function scrubHeaderContent(rawLog: string, options: HeaderScrubOptions): string {
|
||||
const lines = rawLog.split('\n')
|
||||
const out = lines.map((line) => {
|
||||
if (line.trim().length === 0) return line
|
||||
@@ -189,9 +187,9 @@ function scrubHeaderContent(rawLog: string, scrubToolsAndPrefix: boolean): strin
|
||||
const header = data.header as Record<string, unknown> | null | undefined
|
||||
if (header === null || typeof header !== 'object') return line
|
||||
let touched = false
|
||||
if ('system' in header) { header.system = SYSTEM; touched = true }
|
||||
if (scrubToolsAndPrefix && 'tools' in header) { header.tools = TOOLS; touched = true }
|
||||
if (scrubToolsAndPrefix && Array.isArray(header.messagePrefix)) {
|
||||
if (options.system === true && 'system' in header) { header.system = SYSTEM; touched = true }
|
||||
if (options.tools === true && 'tools' in header) { header.tools = TOOLS; touched = true }
|
||||
if (options.prefix === true && Array.isArray(header.messagePrefix)) {
|
||||
header.messagePrefix = header.messagePrefix.map(() => MESSAGE_PREFIX)
|
||||
touched = true
|
||||
}
|
||||
@@ -200,16 +198,16 @@ function scrubHeaderContent(rawLog: string, scrubToolsAndPrefix: boolean): strin
|
||||
if (record.type === 'request/header-delta') {
|
||||
let touched = false
|
||||
const system = data.system as Record<string, unknown> | null | undefined
|
||||
if (system !== null && typeof system === 'object' && Array.isArray(system.insert)) {
|
||||
if (options.system === true && system !== null && typeof system === 'object' && Array.isArray(system.insert)) {
|
||||
system.insert = system.insert.map(() => SYSTEM)
|
||||
touched = true
|
||||
}
|
||||
const tools = data.tools as Record<string, unknown> | null | undefined
|
||||
if (scrubToolsAndPrefix && tools !== null && typeof tools === 'object') {
|
||||
if (options.tools === true && tools !== null && typeof tools === 'object') {
|
||||
if (Array.isArray(tools.added)) { tools.added = tools.added.map(scrubToolSchema); touched = true }
|
||||
if (Array.isArray(tools.changed)) { tools.changed = tools.changed.map(scrubToolSchema); touched = true }
|
||||
}
|
||||
if (scrubToolsAndPrefix && Array.isArray(data.messagePrefix)) {
|
||||
if (options.prefix === true && Array.isArray(data.messagePrefix)) {
|
||||
data.messagePrefix = data.messagePrefix.map(() => MESSAGE_PREFIX)
|
||||
touched = true
|
||||
}
|
||||
|
||||
@@ -1,33 +1,12 @@
|
||||
/**
|
||||
* The ACP snapshot suite factory (REPLAY by default, keyless). A suite is a
|
||||
* scenario table plus a snapshots directory: each scenario under
|
||||
* `<snapshotsDir>/<name>/` ships an `input.json` (the client stdin script) and
|
||||
* a `session.jsonl` fixture; replay boots the real agent subprocess
|
||||
* (./harness.ts), drives it, and diffs the normalized stdout transcript
|
||||
* against the committed `stdout.golden.jsonl`. For model scenarios it ALSO
|
||||
* checks the re-persisted session log — against the `session.jsonl` fixture
|
||||
* itself, not a separate golden: the fixture doubles as the replay source
|
||||
* (recorded scenarios) and the expected produced log (both sides normalized
|
||||
* before comparing).
|
||||
*
|
||||
* Request-header content is pinned by exactly ONE scenario per HEADER CLASS —
|
||||
* scenarios that boot the same config compose the same header. Every JSONL
|
||||
* fixture scrubs the system prompt to `{{system}}`; each class's pinning
|
||||
* scenario stores the readable prompt in `system-prompt.golden.md` and keeps its full
|
||||
* tool schemas in `session.jsonl`, while every other fixture also scrubs tools
|
||||
* to `{{tools}}`. A per-run uniformity guard compares both artifacts against
|
||||
* every live header and forbids unrepresented header deltas (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
|
||||
* in one pass. `pnpm run test:snapshot:refresh` (DSH_SNAPSHOT=refresh) instead
|
||||
* replays the committed model scripts keylessly and writes the current stdout
|
||||
* + persisted-log goldens back without calling a live LLM. The caller resolves
|
||||
* that env into {@link SnapshotSuiteOptions} (env reading stays at the suite
|
||||
* edge, not in this library).
|
||||
* Keyless-by-default ACP snapshot suite factory. Each scenario drives the real subprocess and
|
||||
* compares normalized stdout; comparable session fixtures are both replay input and expected
|
||||
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
|
||||
* mode replays committed scripts and rewrites derived artifacts without a key.
|
||||
*
|
||||
* Exactly one scenario per header-composition class pins the system prompt and tool schemas in
|
||||
* dedicated sidecars. Every live header is checked against that pin, so session-dependent
|
||||
* composition must declare a separate class instead of escaping coverage.
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/suite
|
||||
*/
|
||||
|
||||
@@ -42,11 +21,18 @@ import {
|
||||
normalizeStdout,
|
||||
scrubRequestHeaders,
|
||||
scrubSystemPrompts,
|
||||
scrubToolSchemas,
|
||||
} from './normalize.ts'
|
||||
|
||||
/** The readable system-prompt snapshot beside each header-pinning fixture. */
|
||||
const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.golden.md'
|
||||
|
||||
/** The structured tool-schema snapshot beside each header-pinning fixture. */
|
||||
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.golden.json'
|
||||
|
||||
/** Stable session-log token standing in for the sidecar's initial schemas. */
|
||||
const TOOLS_TOKEN = '{{tools}}'
|
||||
|
||||
/** A snapshot scenario and how its fixtures are produced. */
|
||||
export interface Scenario {
|
||||
name: string
|
||||
@@ -89,20 +75,8 @@ export interface Scenario {
|
||||
*/
|
||||
childSessions?: number
|
||||
/**
|
||||
* Whether THIS scenario pins its header class's model-facing request-header
|
||||
* content. Its actual composed prompt is maintained as a readable
|
||||
* `system-prompt.golden.md`; its JSONL keeps full tool schemas but stores the prompt
|
||||
* as `{{system}}`. Every other scenario of the class stores tools as
|
||||
* `{{tools}}` too ({@link scrubRequestHeaders}). A prompt or tool-schema
|
||||
* change therefore shows up in one focused artifact per class, not every
|
||||
* session fixture. One pin per class suffices because
|
||||
* header composition is class-uniform (parent, spawn child, and fork child
|
||||
* all compose the same prompt-modulo-cwd and the same tools) — and that
|
||||
* premise is ASSERTED, not assumed: every non-pinning run's live headers
|
||||
* must equal its class's pinned fixture's (normalized), so a
|
||||
* session-dependent header (say, a restricted subagent toolset) fails loud
|
||||
* until it gets its own pinning scenario.
|
||||
* Defaults to false.
|
||||
* Whether this scenario is its header class's sole request-header pin. Dedicated sidecars own
|
||||
* the prompt and tool schemas, while every classmate is checked for equality.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
@@ -164,17 +138,9 @@ export function childFixturePaths(dir: string, childSessions: number): string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own
|
||||
* header line (`{ type: 'session', id, cwd }`). A committed fixture carries the
|
||||
* session id and cwd of the run that harvested it — different from the live
|
||||
* replay run — so normalizing it against the live run's ctx would leave those
|
||||
* recorded values unscrubbed. Reading them from the header scrubs the fixture's
|
||||
* own id/cwd to the same `{{sessionId}}`/`{{cwd}}` tokens the replay output gets.
|
||||
* An authored fixture whose header is already normalized (`id:'{{sessionId}}'`,
|
||||
* `cwd:'{{cwd}}'`) yields those tokens as the volatile values, so scrubbing them
|
||||
* is an idempotent no-op. A header with no `cwd` falls back to a sentinel that
|
||||
* cannot occur in a log (NOT `''`, which `String.split` would match on every
|
||||
* character boundary and corrupt the output).
|
||||
* Derive normalization values from a fixture's own session header. Recorded ids and cwd differ
|
||||
* from the live replay run; the non-empty sentinel for missing cwd avoids accidental empty-
|
||||
* string replacement.
|
||||
*
|
||||
* @param fixture The committed `session.jsonl` content.
|
||||
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
|
||||
@@ -225,6 +191,98 @@ export function normalizedSystemPrompts(rawLog: string, ctx: NormalizeContext):
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* The normalized tool-schema arrays carried by request headers in a session
|
||||
* JSONL, in log order. Headers without an array-valued tools field are omitted
|
||||
* so callers can assert one schema set per header explicitly.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content to inspect.
|
||||
* @param ctx The volatile values of the run that produced it.
|
||||
* @returns The normalized initial tool-schema arrays, in header order.
|
||||
*/
|
||||
export function normalizedToolSchemas(rawLog: string, ctx: NormalizeContext): unknown[][] {
|
||||
return normalizedHeaders(rawLog, ctx).flatMap((header) => {
|
||||
if (header === null || typeof header !== 'object') return []
|
||||
const tools = (header as { tools?: unknown }).tools
|
||||
return Array.isArray(tools) ? [tools] : []
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract normalized tool-schema edits from request-header deltas in log order.
|
||||
* Deltas without an object-valued tools edit are omitted; their remaining
|
||||
* structure stays pinned in the session JSONL.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content to inspect.
|
||||
* @param ctx The volatile values of the run that produced it.
|
||||
* @returns The normalized tool-schema edits, in event order.
|
||||
*/
|
||||
export function normalizedToolSchemaDeltas(rawLog: string, ctx: NormalizeContext): unknown[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { tools?: unknown } })
|
||||
.filter(record => record.type === 'request/header-delta')
|
||||
.flatMap((record) => {
|
||||
const tools = record.data?.tools
|
||||
return tools !== null && typeof tools === 'object' && !Array.isArray(tools) ? [tools] : []
|
||||
})
|
||||
}
|
||||
|
||||
/** The structured contents of a tool-schema sidecar. */
|
||||
export interface ToolSchemasSnapshot {
|
||||
/** The complete tool schemas from the pinned request header. */
|
||||
initial: unknown[]
|
||||
/** Complete tool-schema edits from subsequent request-header deltas. */
|
||||
deltas: unknown[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Render tool schemas and later schema edits as canonical, readable JSON.
|
||||
*
|
||||
* @param initial The pinned request header's complete tool schemas.
|
||||
* @param deltas Complete tool-schema edits from request-header deltas.
|
||||
* @returns A pretty-printed JSON snapshot ending in one newline.
|
||||
*/
|
||||
export function formatToolSchemasSnapshot(initial: readonly unknown[], deltas: readonly unknown[] = []): string {
|
||||
return `${JSON.stringify({ initial, deltas }, null, 2)}\n`
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse and validate the stable top-level shape of a tool-schema sidecar.
|
||||
*
|
||||
* @param snapshot The JSON sidecar text.
|
||||
* @returns Its initial schemas and schema deltas.
|
||||
*/
|
||||
export function parseToolSchemasSnapshot(snapshot: string): ToolSchemasSnapshot {
|
||||
const parsed = JSON.parse(snapshot) as unknown
|
||||
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error('acp-snapshot: tool-schema snapshot must be an object')
|
||||
}
|
||||
const { initial, deltas } = parsed as { initial?: unknown; deltas?: unknown }
|
||||
if (!Array.isArray(initial) || !Array.isArray(deltas)) {
|
||||
throw new Error('acp-snapshot: tool-schema snapshot must carry array-valued initial and deltas fields')
|
||||
}
|
||||
return { initial, deltas }
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a sidecar's initial schemas into a tokenized pinned header.
|
||||
*
|
||||
* @param header The parsed request header carrying `tools: "{{tools}}"`.
|
||||
* @param snapshot The parsed tool-schema sidecar.
|
||||
* @returns A copy of the header with its complete initial schemas restored.
|
||||
*/
|
||||
export function restorePinnedToolSchemas(header: unknown, snapshot: ToolSchemasSnapshot): unknown {
|
||||
if (header === null || typeof header !== 'object' || Array.isArray(header)) {
|
||||
throw new Error('acp-snapshot: pinned request header must be an object')
|
||||
}
|
||||
if ((header as { tools?: unknown }).tools !== TOOLS_TOKEN) {
|
||||
throw new Error(`acp-snapshot: pinned request header tools must equal ${TOOLS_TOKEN}`)
|
||||
}
|
||||
return { ...header, tools: snapshot.initial }
|
||||
}
|
||||
|
||||
/** One normalized system-prompt edit carried by a `request/header-delta`. */
|
||||
export interface SystemPromptDeltaSnapshot {
|
||||
/** How many leading lines remain from the prior prompt. */
|
||||
@@ -315,6 +373,27 @@ function parseJsonlRecords(text: string): Record<string, unknown>[] {
|
||||
.map(line => JSON.parse(line) as Record<string, unknown>)
|
||||
}
|
||||
|
||||
/**
|
||||
* Find tool calls whose structured result reports `UNKNOWN_TOOL`.
|
||||
*
|
||||
* Snapshot refresh must not turn a missing registration into accepted behavior;
|
||||
* intentional unknown-tool behavior belongs in a focused unit or e2e test.
|
||||
*
|
||||
* @param rawLog The session JSONL to inspect.
|
||||
* @returns The failing call ids in log order, using a diagnostic placeholder when absent.
|
||||
*/
|
||||
export function unknownToolCallIds(rawLog: string): string[] {
|
||||
return parseJsonlRecords(rawLog).flatMap((record) => {
|
||||
if (record.type !== 'tool/result') return []
|
||||
const data = record.data
|
||||
if (data === null || typeof data !== 'object') return []
|
||||
const { callId, error } = data as { callId?: unknown; error?: unknown }
|
||||
if (error === null || typeof error !== 'object') return []
|
||||
if ((error as { code?: unknown }).code !== 'UNKNOWN_TOOL') return []
|
||||
return [typeof callId === 'string' ? callId : '<missing callId>']
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the cross-log id/cwd replacements used by refresh write-back.
|
||||
*
|
||||
@@ -419,10 +498,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
|
||||
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
|
||||
// REFRESH mode is replay-backed and deterministic, so it runs every
|
||||
// scenario and rewrites the comparable fixtures from that replay run.
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||
// (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
@@ -444,10 +521,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
...scenario.configPath !== undefined ? { configPath: scenario.configPath } : {},
|
||||
})
|
||||
|
||||
// Scrub every volatile id the run produced: the ACP server-issued session
|
||||
// id plus every harvested log's recorded id (a subagent child id never
|
||||
// surfaces over ACP, but it appears in the child's own log header). The
|
||||
// normalizer's UUID catch-all covers any we don't enumerate.
|
||||
for (const log of result.sessionLogs) {
|
||||
expect(unknownToolCallIds(log.content), `session ${log.id}: snapshot scenarios must not accept UNKNOWN_TOOL`)
|
||||
.toEqual([])
|
||||
}
|
||||
|
||||
// Scrub every volatile id the run produced: the ACP server-issued session id plus every
|
||||
// harvested log's recorded id (a subagent child id never surfaces over ACP, but it
|
||||
// appears in the child's own log header).
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: [
|
||||
...result.sessionId !== undefined ? [result.sessionId] : [],
|
||||
@@ -456,17 +537,10 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
|
||||
// live logs back to their fixtures. REFRESH mode does the same from a
|
||||
// keyless replay run for every comparable log, including authored
|
||||
// scenarios that live record deliberately skips. The primary goes to
|
||||
// session.jsonl, each child to session.<n>.jsonl in harvest order. A
|
||||
// Every fixture is written with its system prompt scrubbed. A pinning
|
||||
// scenario keeps the remaining header content (notably tool schemas);
|
||||
// every other scenario scrubs that bulk too. Record/refresh therefore
|
||||
// cannot smuggle prompt text back into JSONL or duplicate schemas.
|
||||
// Record writes live model fixtures; keyless refresh writes every comparable replayed
|
||||
// fixture. Pinning JSONL keeps prefixes but moves prompts and schemas into sidecars.
|
||||
const scrub = scenario.pinsHeader === true
|
||||
? scrubSystemPrompts
|
||||
? (log: string): string => scrubToolSchemas(scrubSystemPrompts(log))
|
||||
: scrubRequestHeaders
|
||||
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
||||
const existingFixtures = REFRESHING
|
||||
@@ -503,6 +577,18 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
normalizedSystemPromptDeltas(primary.content, ctx),
|
||||
)
|
||||
await writeFile(join(dir, SYSTEM_PROMPT_SNAPSHOT), snapshot)
|
||||
|
||||
const schemaSets = result.sessionLogs.flatMap(log => normalizedToolSchemas(log.content, ctx))
|
||||
expect(schemaSets.length, `${mode} produced no tool schemas to snapshot`).toBeGreaterThan(0)
|
||||
const initialSchemaSnapshot = formatToolSchemasSnapshot(schemaSets[0] as unknown[])
|
||||
for (const schemas of schemaSets) {
|
||||
expect(formatToolSchemasSnapshot(schemas), 'the pinning run produced divergent tool schemas')
|
||||
.toEqual(initialSchemaSnapshot)
|
||||
}
|
||||
await writeFile(join(dir, TOOL_SCHEMAS_SNAPSHOT), formatToolSchemasSnapshot(
|
||||
schemaSets[0] as unknown[],
|
||||
normalizedToolSchemaDeltas(primary.content, ctx),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -515,14 +601,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
if (comparesLog) {
|
||||
// The harvested logs (primary-first) must match their committed fixtures
|
||||
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
|
||||
// OWN volatile values — the live run's via `ctx`, the committed fixture's
|
||||
// via its own header (a committed file cannot share the live run's ids).
|
||||
// Both sides pass through the scenario's idempotent scrub: every live
|
||||
// prompt becomes the fixture's `{{system}}`; non-pinning scenarios
|
||||
// additionally tokenize tools/prefix. The dedicated header guard below
|
||||
// compares those omitted values against their class's pin artifacts.
|
||||
// The harvested logs (primary-first) must match their committed fixtures 1:1.
|
||||
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
|
||||
for (let i = 0; i < fixtureFiles.length; i++) {
|
||||
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
||||
@@ -532,11 +611,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
}
|
||||
|
||||
// Header-uniformity guard: every live header in a class must equal the
|
||||
// class pin split across its JSONL header (system token + real tools)
|
||||
// and readable Markdown prompt. A pinning scenario may carry its
|
||||
// declared header deltas; their prompt edits live in the Markdown
|
||||
// golden while JSONL retains the tokenized edit structure.
|
||||
// Header-uniformity guard: every live header in a class must equal the class pin split
|
||||
// across tokenized JSONL plus readable prompt and structured schema sidecars.
|
||||
/* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */
|
||||
const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario
|
||||
const pinningDir = join(snapshotsDir, pinningScenario.name)
|
||||
@@ -544,8 +620,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
const promptSnapshot = await readFile(join(pinningDir, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
|
||||
const initialPromptSnapshot = initialSystemPromptSnapshot(promptSnapshot)
|
||||
const toolSchemasSnapshot = await readFile(join(pinningDir, TOOL_SCHEMAS_SNAPSHOT), 'utf8')
|
||||
const toolSchemas = parseToolSchemasSnapshot(toolSchemasSnapshot)
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
const pinnedHeader = restorePinnedToolSchemas(pinned[0], toolSchemas)
|
||||
for (const [logIndex, log] of result.sessionLogs.entries()) {
|
||||
const expectedDeltas = scenario.pinsHeader === true && logIndex === 0
|
||||
? scenario.expectedHeaderDeltas ?? 0
|
||||
@@ -554,11 +633,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
.toBe(expectedDeltas)
|
||||
const headers = normalizedHeaders(scrubSystemPrompts(log.content), ctx)
|
||||
const prompts = normalizedSystemPrompts(log.content, ctx)
|
||||
const schemaSets = normalizedToolSchemas(log.content, ctx)
|
||||
expect(prompts.length, `session ${log.id}: every request/header must carry a string system prompt`)
|
||||
.toBe(headers.length)
|
||||
expect(schemaSets.length, `session ${log.id}: every request/header must carry an array-valued tools field`)
|
||||
.toBe(headers.length)
|
||||
for (const [k, header] of headers.entries()) {
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
.toEqual(pinnedHeader)
|
||||
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: initial system prompt #${k + 1} diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(initialPromptSnapshot)
|
||||
}
|
||||
@@ -568,6 +650,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
normalizedSystemPromptDeltas(log.content, ctx),
|
||||
), `session ${log.id}: system-prompt deltas diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(promptSnapshot)
|
||||
expect(formatToolSchemasSnapshot(
|
||||
schemaSets[0] as unknown[],
|
||||
normalizedToolSchemaDeltas(log.content, ctx),
|
||||
), `session ${log.id}: tool-schema deltas diverged from ${pinningScenario.name}/${TOOL_SCHEMAS_SNAPSHOT}`)
|
||||
.toEqual(toolSchemasSnapshot)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -586,18 +673,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
})
|
||||
|
||||
it('every registered scenario has its required fixture files', () => {
|
||||
// Every scenario has an input script and an stdout golden. EVERY scenario
|
||||
// also needs `session.jsonl`: the suite boots `llm-replay` with that path
|
||||
// as the replay source for ALL scenarios (the factory passes
|
||||
// `fixtureFile: <dir>/session.jsonl` unconditionally), and `loadReplayScript`
|
||||
// throws "fixture not found" when it is absent and no override replaces it.
|
||||
// A no-model scenario ships a header-only `session.jsonl` (it derives to an
|
||||
// empty script — no model call is made); a model scenario's fixture also
|
||||
// doubles as the expected-log artifact the run is diffed against. The
|
||||
// `replay.override.json` sidecar is matched BOTH ways against the table's
|
||||
// `overridden` flag: required when set, forbidden when not — the harness
|
||||
// forwards the file purely on existence, so an unregistered stray sidecar
|
||||
// would silently replace the derived script.
|
||||
// Every scenario has an input script and an stdout golden.
|
||||
for (const { name, overridden, childSessions, pinsHeader } of scenarios) {
|
||||
const dir = join(snapshotsDir, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
@@ -607,6 +683,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
.toBe(overridden === true)
|
||||
expect(existsSync(join(dir, SYSTEM_PROMPT_SNAPSHOT)), `${name}/${SYSTEM_PROMPT_SNAPSHOT} presence must match \`pinsHeader\``)
|
||||
.toBe(pinsHeader === true)
|
||||
expect(existsSync(join(dir, TOOL_SCHEMAS_SNAPSHOT)), `${name}/${TOOL_SCHEMAS_SNAPSHOT} presence must match \`pinsHeader\``)
|
||||
.toBe(pinsHeader === true)
|
||||
// A nested-agent scenario ships one child fixture per recorded subagent
|
||||
// session (`session.1.jsonl` …), the replay source for that child session.
|
||||
for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
|
||||
@@ -616,10 +694,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
})
|
||||
|
||||
it('exactly one scenario pins the request-header content of each header class', () => {
|
||||
// Zero pins would drop a class's prompt/schema surface from the suite
|
||||
// entirely; two would split it. One pin per class is the design
|
||||
// (pinned-header RFC); WHICH scenario pins is the scenario table's
|
||||
// reviewable choice.
|
||||
// Zero pins would drop a class's prompt/schema surface from the suite entirely; two would
|
||||
// split it.
|
||||
const pins = new Map<string, string[]>()
|
||||
for (const scenario of scenarios.filter(s => s.pinsHeader === true)) {
|
||||
const cls = classOf(scenario)
|
||||
@@ -632,29 +708,32 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
})
|
||||
|
||||
it('every pinning fixture carries one request/header, one readable prompt, and its declared deltas', async () => {
|
||||
// The live uniformity guard runs only in NON-pinning scenarios, so a
|
||||
// class made of just its pinning scenario would otherwise accept a
|
||||
// re-recorded pin with several headers or an undeclared mid-run
|
||||
// header-delta — shapes the pin design cannot represent. Assert the
|
||||
// committed pins directly; a scenario whose arc legitimately rewrites
|
||||
// a prompt section declares the exact count via expectedHeaderDeltas.
|
||||
it('every pinning fixture carries one tokenized request/header, two sidecars, and its declared deltas', async () => {
|
||||
// The live uniformity guard runs only in NON-pinning scenarios, so a class made of just
|
||||
// its pinning scenario would otherwise accept a re-recorded pin with several headers or
|
||||
// an undeclared mid-run header-delta — shapes the pin design cannot represent.
|
||||
for (const scenario of pinningByClass.values()) {
|
||||
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
|
||||
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
|
||||
const promptSnapshot = await readFile(join(snapshotsDir, scenario.name, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
|
||||
const toolSchemasSnapshot = await readFile(join(snapshotsDir, scenario.name, TOOL_SCHEMAS_SNAPSHOT), 'utf8')
|
||||
const toolSchemas = parseToolSchemasSnapshot(toolSchemasSnapshot)
|
||||
expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly one request/header`).toBe(1)
|
||||
expect(() => restorePinnedToolSchemas(headers[0], toolSchemas), `${scenario.name}: tools must use the sidecar token`)
|
||||
.not.toThrow()
|
||||
expect(promptSnapshot.length, `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must not be empty`).toBeGreaterThan(0)
|
||||
expect(promptSnapshot.endsWith('\n'), `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must end in a newline`).toBe(true)
|
||||
expect(toolSchemasSnapshot, `${scenario.name}/${TOOL_SCHEMAS_SNAPSHOT} must use canonical JSON formatting`)
|
||||
.toBe(formatToolSchemasSnapshot(toolSchemas.initial, toolSchemas.deltas))
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared request/header-deltas`)
|
||||
.toBe(scenario.expectedHeaderDeltas ?? 0)
|
||||
}
|
||||
})
|
||||
|
||||
it('every committed JSONL omits system prompts and only pinning fixtures keep other header bulk', async () => {
|
||||
// System prompts always live in the readable Markdown artifact. Header
|
||||
// pins keep tool schemas/prefixes in JSONL; every other fixture tokenizes
|
||||
// all header bulk. Fixed-point checks make both storage rules fail loud.
|
||||
it('every committed JSONL has valid tool results and canonical header storage', async () => {
|
||||
// Prompts and schemas always leave JSONL. Header pins retain prefixes;
|
||||
// every other fixture tokenizes those too. Fixed-point checks make both
|
||||
// storage rules fail loud.
|
||||
for (const scenario of scenarios) {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const files = [
|
||||
@@ -663,12 +742,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
]
|
||||
for (const file of files) {
|
||||
const fixture = await readFile(join(dir, file), 'utf8')
|
||||
expect(unknownToolCallIds(fixture), `${scenario.name}/${file} contains UNKNOWN_TOOL`)
|
||||
.toEqual([])
|
||||
expect(scrubSystemPrompts(fixture), `${scenario.name}/${file} carries an unscrubbed system prompt`)
|
||||
.toEqual(fixture)
|
||||
if (scenario.pinsHeader === true) {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must pin the non-system header content`)
|
||||
.not.toEqual(fixture)
|
||||
} else {
|
||||
expect(scrubToolSchemas(fixture), `${scenario.name}/${file} carries unscrubbed tool schemas`)
|
||||
.toEqual(fixture)
|
||||
if (scenario.pinsHeader !== true) {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
||||
.toEqual(fixture)
|
||||
}
|
||||
|
||||
@@ -1,19 +1,7 @@
|
||||
/**
|
||||
* Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs. Speaks
|
||||
* newline-delimited JSON-RPC on stdio like the real `dsh-acp-agent` bin, but
|
||||
* every behavior — how prompts settle, whether session/new rejects, which
|
||||
* session logs get persisted, what filesystem noise to leave — comes from a
|
||||
* `behavior.json` sitting NEXT to the `$DSH_SNAPSHOT_FILE` fixture, so a spec
|
||||
* scripts a whole subprocess run from data. The specs launch it through the
|
||||
* REAL `runScenario` spawn path (tsx loader, temp cwd, env plumbing), so the
|
||||
* harness plumbing is exercised for real; only the agent behind the protocol
|
||||
* is scripted.
|
||||
*
|
||||
* The specs (not the golden tier) own this bin: it asserts nothing, echoes
|
||||
* observable facts into `session/update` text chunks (env probe, permission
|
||||
* outcome, seeded-workspace listing) for the spec to read off `rawStdout`, and
|
||||
* exits 0 on stdin EOF after writing the scripted logs — mirroring the real
|
||||
* bin's dispose-flush-exit shape.
|
||||
* Scripted ACP agent for snapshot-kit tests. A fixture-adjacent `behavior.json` controls the
|
||||
* subprocess reached through the real harness path; the bin reports observations over ACP and
|
||||
* writes scripted logs before exiting on stdin EOF.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
{"type":"session","id":"ccdc749f-56f3-4267-9750-598b5c60b7b2","createdAt":600,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-nOQ4Gy"}
|
||||
{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
|
||||
{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"initial": [
|
||||
{
|
||||
"name": "t1",
|
||||
"description": "D1",
|
||||
"parameters": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"deltas": []
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"}
|
||||
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
|
||||
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
{"type":"request/header-delta","seq":1,"time":7,"data":{"system":{"keepStart":1,"keepEnd":0,"insert":["{{system}}"]}}}
|
||||
{"type":"turn/start","seq":2,"time":7,"data":{"turn":1}}
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"initial": [
|
||||
{
|
||||
"name": "t1",
|
||||
"description": "D1",
|
||||
"parameters": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"deltas": []
|
||||
}
|
||||
@@ -308,11 +308,8 @@ describe('runScenario', () => {
|
||||
|
||||
it('rejects the run on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ permissionProbe: true })
|
||||
// The fake bin offers allow_once/reject_once; scripting allow_always is a
|
||||
// scenario bug. The agent is answered `cancelled` (it must not be able to
|
||||
// absorb the bug as an error-means-denial), and the RUN fails: a callback
|
||||
// throw would only reach the agent as a JSON-RPC error response, letting
|
||||
// a tolerant agent carry on and the scenario pass — or record.
|
||||
// The fake offers only allow_once/reject_once. The harness must reject an impossible click,
|
||||
// not merely send an RPC error that a tolerant agent could absorb.
|
||||
await expect(runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
normalizeStdout,
|
||||
scrubRequestHeaders,
|
||||
scrubSystemPrompts,
|
||||
scrubToolSchemas,
|
||||
} from '../src/normalize.ts'
|
||||
|
||||
/**
|
||||
@@ -352,3 +353,45 @@ describe('scrubSystemPrompts', () => {
|
||||
expect(scrubSystemPrompts(out)).toBe(out)
|
||||
})
|
||||
})
|
||||
|
||||
describe('scrubToolSchemas', () => {
|
||||
it('scrubs only tool-schema payloads while keeping prompts and prefixes verbatim', () => {
|
||||
const header = JSON.stringify({
|
||||
type: 'request/header', seq: 1, time: 2,
|
||||
data: {
|
||||
header: {
|
||||
system: 'full prompt',
|
||||
tools: [{ name: 'read', description: 'full schema', parameters: { type: 'object' } }],
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'full prefix' }] }],
|
||||
},
|
||||
reason: 'initial',
|
||||
},
|
||||
})
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 2, time: 3,
|
||||
data: {
|
||||
system: { keepStart: 1, keepEnd: 2, insert: ['new prompt line'] },
|
||||
tools: { added: [{ name: 'grep', description: 'new schema' }], changed: [{ name: 'read', description: 'changed schema' }] },
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
},
|
||||
})
|
||||
const systemOnly = JSON.stringify({
|
||||
type: 'request/header', seq: 3, time: 4,
|
||||
data: { header: { system: 'prompt only' }, reason: 'resume' },
|
||||
})
|
||||
|
||||
const out = scrubToolSchemas(`${header}\n${delta}\n${systemOnly}\n`)
|
||||
expect(out).toContain('"tools":"{{tools}}"')
|
||||
expect(out).toContain('"added":[{"name":"grep","description":"{{tools}}"}]')
|
||||
expect(out).toContain('"changed":[{"name":"read","description":"{{tools}}"}]')
|
||||
expect(out).not.toContain('full schema')
|
||||
expect(out).not.toContain('new schema')
|
||||
expect(out).not.toContain('changed schema')
|
||||
expect(out).toContain('full prompt')
|
||||
expect(out).toContain('new prompt line')
|
||||
expect(out).toContain('full prefix')
|
||||
expect(out).toContain('changed prefix')
|
||||
expect(out.split('\n')[2]).toBe(systemOnly)
|
||||
expect(scrubToolSchemas(out)).toBe(out)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -9,30 +9,29 @@ import {
|
||||
childFixturePaths,
|
||||
fixtureContext,
|
||||
formatSystemPromptSnapshot,
|
||||
formatToolSchemasSnapshot,
|
||||
headerDeltaCount,
|
||||
normalizedHeaders,
|
||||
normalizedSystemPromptDeltas,
|
||||
normalizedSystemPrompts,
|
||||
normalizedToolSchemaDeltas,
|
||||
normalizedToolSchemas,
|
||||
parseToolSchemasSnapshot,
|
||||
refreshFixtureReplacements,
|
||||
restorePinnedToolSchemas,
|
||||
stabilizeRefreshLog,
|
||||
unknownToolCallIds,
|
||||
} from '../src/suite.ts'
|
||||
|
||||
/**
|
||||
* Unit tests for the suite factory, by running it: two synthetic suites over
|
||||
* the scripted fake ACP bin (./fixtures/fake-acp-agent.ts) register REAL
|
||||
* describe/it trees at collection time, so every factory path — golden and log
|
||||
* compares, the per-suite header pin and its uniformity guard, record-mode
|
||||
* fixture write-back, skip semantics, and the fixture guard block — executes
|
||||
* as an ordinary green test. The pure helpers get direct cases below.
|
||||
* Unit tests for the suite factory, by running it: two synthetic suites over the scripted fake
|
||||
* ACP bin (./fixtures/fake-acp-agent.ts) register real describe/it trees at collection time,
|
||||
* so every factory path — golden and log compares, the per-suite header pin and its uniformity
|
||||
* guard, record-mode fixture write-back, skip semantics, and the fixture guard block —
|
||||
* executes as an ordinary green test.
|
||||
*
|
||||
* The replay suite runs against the committed fixtures in ./fixtures/suite.
|
||||
* The record suite runs against a TEMP COPY of ./fixtures/record-suite
|
||||
* (record mode writes session fixtures back into its snapshots dir; a run must
|
||||
* never touch the committed tree). To re-bootstrap the record tree's goldens
|
||||
* after changing the fake bin's output, run this spec once with
|
||||
* `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1` (points the record suite at the committed
|
||||
* tree so vitest creates/updates the goldens and the write-back lands there),
|
||||
* then commit the result.
|
||||
* Record tests use a temp copy. To intentionally rebuild their committed fixtures, run this
|
||||
* spec once with `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1`, then review and commit the resulting tree.
|
||||
*/
|
||||
|
||||
const AGENT = {
|
||||
@@ -44,12 +43,7 @@ const AGENT = {
|
||||
const REPLAY_DIR = fileURLToPath(new URL('./fixtures/suite', import.meta.url))
|
||||
const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.url))
|
||||
|
||||
// The replay suite doubles as the header-CLASS coverage: every scenario names
|
||||
// the same explicit class (the record suite exercises the 'default' fallback),
|
||||
// and plain-turn boots through a per-scenario configPath override (the same
|
||||
// dummy path the agent default carries — the plumbing, not the composition,
|
||||
// is what this suite can exercise; the real overlay boot is the acp-agent
|
||||
// example's code-mode scenarios).
|
||||
// Replay pins explicit header classes; recording covers the default fallback.
|
||||
const REPLAY_SCENARIOS: Scenario[] = [
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderDeltas: 1, headerClass: 'main' },
|
||||
{ name: 'plain-turn', hasModelTurn: true, recorded: true, childSessions: 1, headerClass: 'main', configPath: AGENT.configPath },
|
||||
@@ -82,6 +76,7 @@ afterAll(async () => {
|
||||
function staleRefreshFixtures(dir: string): void {
|
||||
writeFileSync(join(dir, 'plain-turn', 'stdout.golden.jsonl'), 'stale stdout\n')
|
||||
writeFileSync(join(dir, 'pin-turn', 'system-prompt.golden.md'), 'STALE PROMPT\n')
|
||||
writeFileSync(join(dir, 'pin-turn', 'tool-schemas.golden.json'), '{"initial":[{"name":"stale"}],"deltas":[]}\n')
|
||||
|
||||
const plainBehaviorFile = join(dir, 'plain-turn', 'behavior.json')
|
||||
const plainBehavior = JSON.parse(readFileSync(plainBehaviorFile, 'utf8')) as Record<string, unknown>
|
||||
@@ -137,6 +132,9 @@ describe('defineAcpSnapshotSuite: refresh write-back', () => {
|
||||
'NEW PROMPT LINE',
|
||||
'',
|
||||
].join('\n'))
|
||||
const schemas = readFileSync(join(refreshDir, 'pin-turn', 'tool-schemas.golden.json'), 'utf8')
|
||||
expect(schemas).toContain('"description": "D1"')
|
||||
expect(schemas).not.toContain('"name":"stale"')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -247,6 +245,40 @@ describe('normalizedSystemPrompts', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedToolSchemas', () => {
|
||||
it('extracts normalized schema arrays and omits absent or non-array fields', () => {
|
||||
const log = [
|
||||
'{"type":"session","id":"a","createdAt":5,"cwd":"/w"}',
|
||||
'{"type":"request/header","seq":0,"time":9,"data":{"header":{"tools":[{"name":"read","description":"work in /w"}]}}}',
|
||||
'{"type":"request/header","seq":1,"time":9,"data":{"header":{}}}',
|
||||
'{"type":"request/header","seq":2,"time":9,"data":{"header":{"tools":null}}}',
|
||||
'{"type":"request/header","seq":3,"time":9,"data":{"header":null}}',
|
||||
'{"type":"request/header","seq":4,"time":9,"data":{"header":"invalid"}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(normalizedToolSchemas(log, { sessionIds: [], cwd: '/w' })).toEqual([
|
||||
[{ name: 'read', description: 'work in {{cwd}}' }],
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedToolSchemaDeltas', () => {
|
||||
it('extracts and normalizes object-valued schema edits', () => {
|
||||
const log = [
|
||||
'{"type":"request/header-delta","data":{"tools":{"added":[{"name":"read","description":"work in /w"}]}}}',
|
||||
'{"type":"request/header-delta","data":{"tools":null}}',
|
||||
'{"type":"request/header-delta","data":{"tools":"invalid"}}',
|
||||
'{"type":"request/header-delta","data":{"tools":[]}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"insert":[]}}}',
|
||||
'{"type":"request/header","data":{"tools":{"added":[]}}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(normalizedToolSchemaDeltas(log, { sessionIds: [], cwd: '/w' })).toEqual([
|
||||
{ added: [{ name: 'read', description: 'work in {{cwd}}' }] },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedSystemPromptDeltas', () => {
|
||||
it('extracts and normalizes well-formed system edits', () => {
|
||||
const log = [
|
||||
@@ -281,6 +313,39 @@ describe('formatSystemPromptSnapshot', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('tool-schema snapshots', () => {
|
||||
const snapshot = {
|
||||
initial: [{ name: 'read', description: 'Read a file.' }],
|
||||
deltas: [{ added: [{ name: 'grep', description: 'Search files.' }] }],
|
||||
}
|
||||
|
||||
it('formats and parses canonical structured JSON', () => {
|
||||
const formatted = formatToolSchemasSnapshot(snapshot.initial, snapshot.deltas)
|
||||
expect(formatted).toBe(`${JSON.stringify(snapshot, null, 2)}\n`)
|
||||
expect(parseToolSchemasSnapshot(formatted)).toEqual(snapshot)
|
||||
})
|
||||
|
||||
it('rejects invalid top-level and field shapes', () => {
|
||||
expect(() => parseToolSchemasSnapshot('null')).toThrow(/must be an object/)
|
||||
expect(() => parseToolSchemasSnapshot('"invalid"')).toThrow(/must be an object/)
|
||||
expect(() => parseToolSchemasSnapshot('[]')).toThrow(/must be an object/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":{},"deltas":[]}')).toThrow(/array-valued/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":[],"deltas":{}}')).toThrow(/array-valued/)
|
||||
})
|
||||
|
||||
it('restores initial schemas into the pinned header token', () => {
|
||||
expect(restorePinnedToolSchemas({ system: '{{system}}', tools: '{{tools}}' }, snapshot))
|
||||
.toEqual({ system: '{{system}}', tools: snapshot.initial })
|
||||
})
|
||||
|
||||
it('rejects invalid headers and a missing tool token', () => {
|
||||
expect(() => restorePinnedToolSchemas(null, snapshot)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas('invalid', snapshot)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas([], snapshot)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas({ tools: [] }, snapshot)).toThrow(/must equal/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('headerDeltaCount', () => {
|
||||
it('counts request/header-delta events, ignoring blanks and other lines', () => {
|
||||
const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} })
|
||||
@@ -290,6 +355,27 @@ describe('headerDeltaCount', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('unknownToolCallIds', () => {
|
||||
it('returns structured UNKNOWN_TOOL call ids and ignores other results', () => {
|
||||
const log = [
|
||||
'{"type":"tool/result","data":{"callId":"missing","error":{"code":"UNKNOWN_TOOL"}}}',
|
||||
'{"type":"tool/result","data":{"callId":"failed","error":{"code":"EXECUTION_FAILED"}}}',
|
||||
'{"type":"tool/result","data":null}',
|
||||
'{"type":"tool/result","data":"invalid"}',
|
||||
'{"type":"tool/result","data":{"error":null}}',
|
||||
'{"type":"tool/result","data":{"error":"invalid"}}',
|
||||
'{"type":"assistant/message","data":{"error":{"code":"UNKNOWN_TOOL"}}}',
|
||||
'{"type":"tool/result","data":{"error":{"code":"UNKNOWN_TOOL"}}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(unknownToolCallIds(log)).toEqual(['missing', '<missing callId>'])
|
||||
})
|
||||
|
||||
it('returns no failures for ordinary tool results', () => {
|
||||
expect(unknownToolCallIds('{"type":"tool/result","data":{"callId":"ok"}}\n')).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('refreshFixtureReplacements', () => {
|
||||
it('maps fresh ids and cwd values to the existing fixture values, skipping non-replacements', () => {
|
||||
const log = (content: string): HarvestedLog => ({ id: 'diagnostic', createdAt: 1, content })
|
||||
|
||||
Reference in New Issue
Block a user