test(acp): await cancelled tool updates

This commit is contained in:
Tianyi Cui
2026-07-19 13:09:16 +08:00
parent b99008221e
commit d709a8a1a4
4 changed files with 45 additions and 7 deletions
@@ -5,7 +5,8 @@
{ {
"op": "promptAndCancel", "op": "promptAndCancel",
"text": "Run two shell commands: wait for cancellation, then write skipped.txt.", "text": "Run two shell commands: wait for cancellation, then write skipped.txt.",
"afterUpdate": "tool_call" "afterUpdate": "tool_call",
"waitForToolCallUpdate": "call_skipped"
} }
] ]
} }
+13 -3
View File
@@ -39,8 +39,8 @@ export type { AgentUnderTest } from './launcher.ts'
* *
* `promptAndCancel` starts a prompt without awaiting completion, waits until * `promptAndCancel` starts a prompt without awaiting completion, waits until
* the client observes the selected update (`agent_message_chunk` by default), * the client observes the selected update (`agent_message_chunk` by default),
* then cancels and awaits completion. This keeps update/cancel order * then cancels and awaits completion. A named `waitForToolCallUpdate` keeps the
* deterministic for fixtures that a plain `prompt` cannot drive. * step open for a terminal tool update that may follow the prompt response.
*/ */
export type InputStep = export type InputStep =
| { op: 'initialize'; terminalOutput?: boolean } | { op: 'initialize'; terminalOutput?: boolean }
@@ -48,7 +48,12 @@ export type InputStep =
| { op: 'newSessionExpectError'; additionalDirectories?: string[] } | { op: 'newSessionExpectError'; additionalDirectories?: string[] }
| { op: 'prompt'; text: string } | { op: 'prompt'; text: string }
| { op: 'promptExpectError'; text: string } | { op: 'promptExpectError'; text: string }
| { op: 'promptAndCancel'; text: string; afterUpdate?: 'agent_message_chunk' | 'tool_call' } | {
op: 'promptAndCancel'
text: string
afterUpdate?: 'agent_message_chunk' | 'tool_call'
waitForToolCallUpdate?: string
}
| { op: 'cancel' } | { op: 'cancel' }
| { op: 'setConfigOption'; configId: string; value: string } | { op: 'setConfigOption'; configId: string; value: string }
| { op: 'setConfigOptionExpectError'; configId: string; value: string } | { op: 'setConfigOptionExpectError'; configId: string; value: string }
@@ -347,8 +352,13 @@ async function runStep(
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] }) const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
const afterUpdate = step.afterUpdate ?? 'agent_message_chunk' const afterUpdate = step.afterUpdate ?? 'agent_message_chunk'
await waitForUpdate(u => u.sessionUpdate === afterUpdate) await waitForUpdate(u => u.sessionUpdate === afterUpdate)
// Arm this before cancellation so a fast tool drain cannot outrun the waiter.
const toolCallUpdateDone = step.waitForToolCallUpdate === undefined
? undefined
: waitForUpdate(u => u.sessionUpdate === 'tool_call_update' && u.toolCallId === step.waitForToolCallUpdate)
await client.cancel({ sessionId }) await client.cancel({ sessionId })
await promptDone await promptDone
if (toolCallUpdateDone !== undefined) await toolCallUpdateDone
return return
} }
case 'cancel': { case 'cancel': {
@@ -35,6 +35,8 @@ interface Behavior {
prompt?: 'respond' | 'error' | 'hang-until-cancel' prompt?: 'respond' | 'error' | 'hang-until-cancel'
/** Emit a tool call instead of a message chunk before parking a cancellable prompt. */ /** Emit a tool call instead of a message chunk before parking a cancellable prompt. */
cancelAtToolCall?: boolean cancelAtToolCall?: boolean
/** Emit the parked tool call's terminal update after answering cancellation. */
cancelToolCallUpdate?: boolean
/** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */ /** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */
permissionProbe?: boolean permissionProbe?: boolean
/** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */ /** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */
@@ -247,6 +249,19 @@ function handleFrame(frame: Record<string, unknown>): void {
const parked = parkedPromptId const parked = parkedPromptId
parkedPromptId = null parkedPromptId = null
respond(parked, { stopReason: 'cancelled' }) respond(parked, { stopReason: 'cancelled' })
if (behavior.cancelToolCallUpdate === true) {
send({
method: 'session/update',
params: {
sessionId,
update: {
sessionUpdate: 'tool_call_update',
toolCallId: 'call_fake_1',
status: 'failed',
},
},
})
}
} }
return return
default: default:
@@ -334,14 +334,26 @@ describe('runScenario', () => {
expect(result.rawStdout.indexOf('thinking about it')).toBeLessThan(result.rawStdout.indexOf('cancelled')) expect(result.rawStdout.indexOf('thinking about it')).toBeLessThan(result.rawStdout.indexOf('cancelled'))
}) })
it('promptAndCancel can wait for a tool call before cancelling', { timeout: 20_000 }, async () => { it('promptAndCancel can bracket cancellation with tool-call updates', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ prompt: 'hang-until-cancel', cancelAtToolCall: true }) const { fixtureFile } = await scenario({
prompt: 'hang-until-cancel',
cancelAtToolCall: true,
cancelToolCallUpdate: true,
})
const result = await runScenario( const result = await runScenario(
{ steps: [...boot, { op: 'promptAndCancel', text: 'hang', afterUpdate: 'tool_call' }] }, {
steps: [...boot, {
op: 'promptAndCancel',
text: 'hang',
afterUpdate: 'tool_call',
waitForToolCallUpdate: 'call_fake_1',
}],
},
{ agent: AGENT, mode: 'replay', fixtureFile }, { agent: AGENT, mode: 'replay', fixtureFile },
) )
expect(result.rawStdout).toContain('"sessionUpdate":"tool_call"') expect(result.rawStdout).toContain('"sessionUpdate":"tool_call"')
expect(result.rawStdout.indexOf('"sessionUpdate":"tool_call"')).toBeLessThan(result.rawStdout.indexOf('cancelled')) expect(result.rawStdout.indexOf('"sessionUpdate":"tool_call"')).toBeLessThan(result.rawStdout.indexOf('cancelled'))
expect(result.rawStdout.indexOf('cancelled')).toBeLessThan(result.rawStdout.indexOf('"sessionUpdate":"tool_call_update"'))
}) })
it('promptExpectError swallows a model-error response as the expected outcome', { timeout: 20_000 }, async () => { it('promptExpectError swallows a model-error response as the expected outcome', { timeout: 20_000 }, async () => {