test(acp-example): five snapshot scenarios + cancel/error input ops

Adds the first cut of snapshot scenarios, each asserting a normalized stdout
transcript golden and (for model turns) a re-persisted session-log golden:

- text-turn, tool-call-turn, multi-turn: RECORDED against the real API — the
  committed session.jsonl is a genuine harvested log; replay derives the model
  script from it and reproduces deterministically with no key. tool-call-turn
  exercises the real bash executor (echo SNAPSHOT_OK → tool/call + tool/result
  + a post-tool answer step).
- error-finish, cancel: AUTHORED via a replay.override.json sidecar (the live
  API can't be coaxed into a deterministic 401 or mid-stream cancel). error-
  finish replays a {kind:throw} 401 → the bridge answers the prompt with a
  JSON-RPC error and the log records turn/end{kind:error}; cancel replays a
  {kind:hang} → stopReason:cancelled.

Two input-DSL ops support these: promptExpectError (awaits the prompt, asserts
it rejects — the editor's view of a failed turn — and swallows it) and
promptAndCancel (dispatches the prompt unawaited, waits until the client
OBSERVES the streamed agent_message_chunk, then cancels — pinning frame order
so the cancel transcript is deterministic; fixes a flake Codex caught where the
late chunk and the cancelled response could interleave either way).

Scenarios carry a `recorded` flag so test:snapshot:record only re-runs the
live-API ones. reasoning/max-tokens scenarios are deferred (hard to force
deterministically from the live model). Per docs/rfc/implemented/2026-06-19.
This commit is contained in:
Tianyi Cui
2026-06-19 04:10:49 +08:00
parent 81d434896d
commit c94f1563f5
24 changed files with 5173 additions and 9 deletions
+54 -2
View File
@@ -44,11 +44,19 @@ const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta
* harness interprets these in order. `newSession` captures the server-issued
* (random) session id into a `{{sessionId}}` variable that later steps
* reference, since a committed file cannot know the id in advance.
*
* `promptAndCancel` sends a prompt WITHOUT awaiting its response, waits until
* the client observes the first streamed `agent_message_chunk` (so the emitted
* frames deterministically precede the cancellation), then cancels the turn —
* the only way to exercise a cancel deterministically (a plain `prompt` step
* awaits the response, which a cancel/hang scenario would block on forever).
*/
type InputStep =
| { op: 'initialize'; terminalOutput?: boolean }
| { op: 'newSession' }
| { op: 'prompt'; text: string }
| { op: 'promptExpectError'; text: string }
| { op: 'promptAndCancel'; text: string }
| { op: 'cancel' }
/** A scenario's `input.json`: an ordered list of input steps. */
@@ -122,8 +130,23 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
Readable.toWeb(passthrough) as ReadableStream<Uint8Array>,
)
// Watcher so a step can block until the client OBSERVES a particular
// session/update — used by promptAndCancel to pin frame order (send cancel
// only after the streamed agent_message_chunk has arrived, so those frames
// deterministically precede the cancelled prompt response).
const updateWaiters: { match: (u: SessionNotification['update']) => boolean; resolve: () => void }[] = []
const waitForUpdate = (match: (u: SessionNotification['update']) => boolean): Promise<void> =>
new Promise<void>(resolve => updateWaiters.push({ match, resolve }))
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(_params: SessionNotification): Promise<void> {
sessionUpdate(params: SessionNotification): Promise<void> {
for (let i = updateWaiters.length - 1; i >= 0; i--) {
const waiter = updateWaiters[i]
if (waiter !== undefined && waiter.match(params.update)) {
updateWaiters.splice(i, 1)
waiter.resolve()
}
}
return Promise.resolve()
},
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
@@ -136,7 +159,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
let sessionLog: string | undefined
try {
for (const step of input.steps) {
await runStep(client, step, cwd, () => sessionId, (id) => { sessionId = id })
await runStep(client, step, cwd, waitForUpdate, () => sessionId, (id) => { sessionId = id })
}
// Done driving: close stdin so the server disposes gracefully (flushing
// persistence) and exits. Then await exit so the harvested log is complete.
@@ -170,6 +193,7 @@ async function runStep(
client: ClientSideConnection,
step: InputStep,
cwd: string,
waitForUpdate: (match: (u: SessionNotification['update']) => boolean) => Promise<void>,
getSessionId: () => string | undefined,
setSessionId: (id: string) => void,
): Promise<void> {
@@ -191,6 +215,34 @@ async function runStep(
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
return
}
case 'promptExpectError': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: promptExpectError before newSession')
// The model fails this turn (a recorded provider error), so the bridge
// answers the prompt with a JSON-RPC error and the SDK rejects. That
// rejection IS the expected editor experience — swallow it so the run
// completes and the stdout transcript (the error frame) is captured.
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
.then(() => { throw new Error('snapshot-harness: expected the prompt to fail but it succeeded') },
() => { /* expected: the turn failed and the bridge returned an error */ })
return
}
case 'promptAndCancel': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on
// its own). To pin frame order deterministically, wait until the client
// has OBSERVED the hang's streamed agent_message_chunk before cancelling —
// so those update frames always precede the cancelled prompt response in
// the transcript (without this, the late chunk and the response race; see
// the Codex review of commit 5). Then cancel and await the prompt, which
// the bridge settles as `cancelled` once the abort propagates.
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
await client.cancel({ sessionId })
await promptDone
return
}
case 'cancel': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession')