test(acp-snapshot): skip POSIX-cancel scenarios on Windows
The cancel-tool-calls scenario cancels a live bash call, which relies on POSIX detached-process-group termination; bash has no Windows process-tree kill yet (deferred with the Bash execution domain), so the hung call times the scenario out on the native Windows snapshot lane. Add a posixOnly scenario declaration that skips the run test on win32 while the fixture guards keep covering committed files on every platform, and mark cancel-tool-calls with it.
This commit is contained in:
@@ -111,7 +111,9 @@ const SCENARIOS: Scenario[] = [
|
|||||||
configPath: WORKSPACE_CONTEXT_CONFIG,
|
configPath: WORKSPACE_CONTEXT_CONFIG,
|
||||||
},
|
},
|
||||||
{ name: 'cancel', hasModelTurn: true, recorded: false, overridden: true },
|
{ name: 'cancel', hasModelTurn: true, recorded: false, overridden: true },
|
||||||
{ name: 'cancel-tool-calls', hasModelTurn: true, recorded: false, overridden: true },
|
// Cancelling a live bash call relies on POSIX process-group termination;
|
||||||
|
// Windows bash process-tree kill is deferred with the Bash execution domain.
|
||||||
|
{ name: 'cancel-tool-calls', hasModelTurn: true, recorded: false, overridden: true, posixOnly: true },
|
||||||
{ name: 'subagent-spawn', hasModelTurn: true, recorded: true },
|
{ name: 'subagent-spawn', hasModelTurn: true, recorded: true },
|
||||||
{ name: 'subagent-multi', hasModelTurn: true, recorded: true },
|
{ name: 'subagent-multi', hasModelTurn: true, recorded: true },
|
||||||
{ name: 'subagent-fork', hasModelTurn: true, recorded: true },
|
{ name: 'subagent-fork', hasModelTurn: true, recorded: true },
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ defineAcpSnapshotSuite({
|
|||||||
|
|
||||||
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode and filesystem scenarios are templates. Each pinning directory stores the normalized full prompt sequence in generated `system-prompt.golden.md` and the corresponding full tool-schema sequence in generated `tool-schemas.golden.json`; `session.jsonl` stores `"system":"{{system}}","tools":"{{tools}}"` while retaining config, reason, and any model-visible prefix. A pin with legitimate mid-run header changes declares `expectedHeaderChanges`, which fixes the length of both sidecar sequences.
|
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode and filesystem scenarios are templates. Each pinning directory stores the normalized full prompt sequence in generated `system-prompt.golden.md` and the corresponding full tool-schema sequence in generated `tool-schemas.golden.json`; `session.jsonl` stores `"system":"{{system}}","tools":"{{tools}}"` while retaining config, reason, and any model-visible prefix. A pin with legitimate mid-run header changes declares `expectedHeaderChanges`, which fixes the length of both sidecar sequences.
|
||||||
|
|
||||||
Every scenario compares `stdout.golden.jsonl` with cwd-rooted separators canonicalized to `/`. On Windows, `pinsNativeWindowsStdout` additionally compares the complete `stdout.golden.windows.jsonl` after the shared golden and requires that sidecar exactly when enabled.
|
Every scenario compares `stdout.golden.jsonl` with cwd-rooted separators canonicalized to `/`. On Windows, `pinsNativeWindowsStdout` additionally compares the complete `stdout.golden.windows.jsonl` after the shared golden and requires that sidecar exactly when enabled. A scenario whose driven behavior needs POSIX process semantics (e.g. cancelling a live bash call kills a detached process group) declares `posixOnly`, which skips its run test on Windows while the fixture guards keep covering its committed files everywhere.
|
||||||
|
|
||||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log goldens, and each pin's prompt and tool-schema sidecars from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log goldens, and each pin's prompt and tool-schema sidecars from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||||
|
|
||||||
|
|||||||
@@ -111,6 +111,32 @@ export interface Scenario {
|
|||||||
* exactly when the option is set.
|
* exactly when the option is set.
|
||||||
*/
|
*/
|
||||||
pinsNativeWindowsStdout?: boolean
|
pinsNativeWindowsStdout?: boolean
|
||||||
|
/**
|
||||||
|
* Whether the driven behavior needs POSIX process semantics the harness
|
||||||
|
* cannot exercise on Windows (e.g. cancelling a live bash tool call kills a
|
||||||
|
* detached process group). The scenario's run test is skipped on Windows;
|
||||||
|
* its fixtures stay guarded on every platform.
|
||||||
|
*/
|
||||||
|
posixOnly?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether a scenario's run test is skipped for this mode and host: record mode
|
||||||
|
* skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly}
|
||||||
|
* scenarios skip on Windows.
|
||||||
|
*
|
||||||
|
* @param scenario The scenario whose run test is being registered.
|
||||||
|
* @param recording Whether the suite runs in record mode.
|
||||||
|
* @param platform The running Node platform, injectable for unit coverage.
|
||||||
|
* @returns True when the scenario's run test must not execute.
|
||||||
|
*/
|
||||||
|
export function scenarioSkipped(
|
||||||
|
scenario: Scenario,
|
||||||
|
recording: boolean,
|
||||||
|
platform: NodeJS.Platform = process.platform,
|
||||||
|
): boolean {
|
||||||
|
if (recording && !scenario.recorded) return true
|
||||||
|
return scenario.posixOnly === true && platform === 'win32'
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One stdout golden selected for a platform run. */
|
/** One stdout golden selected for a platform run. */
|
||||||
@@ -499,8 +525,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|||||||
scenarioSuite('snapshot scenarios', () => {
|
scenarioSuite('snapshot scenarios', () => {
|
||||||
for (const scenario of scenarios) {
|
for (const scenario of scenarios) {
|
||||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||||
// (sidecar-driven errors/cancel) are never re-recorded.
|
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on
|
||||||
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
|
// Windows, where their process semantics cannot be driven.
|
||||||
|
it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
|
||||||
const dir = join(snapshotsDir, scenario.name)
|
const dir = join(snapshotsDir, scenario.name)
|
||||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||||
const overrideFile = join(dir, 'replay.override.json')
|
const overrideFile = join(dir, 'replay.override.json')
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
normalizedToolSchemas,
|
normalizedToolSchemas,
|
||||||
parseToolSchemasSnapshot,
|
parseToolSchemasSnapshot,
|
||||||
refreshFixtureReplacements,
|
refreshFixtureReplacements,
|
||||||
|
scenarioSkipped,
|
||||||
sessionFixtureNames,
|
sessionFixtureNames,
|
||||||
restorePinnedToolSchemas,
|
restorePinnedToolSchemas,
|
||||||
stabilizeRefreshLog,
|
stabilizeRefreshLog,
|
||||||
@@ -256,6 +257,23 @@ describe('stdoutGoldenVariants', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('scenarioSkipped', () => {
|
||||||
|
const authored: Scenario = { name: 'authored', hasModelTurn: true, recorded: false }
|
||||||
|
const posix: Scenario = { name: 'posix-cancel', hasModelTurn: true, recorded: false, posixOnly: true }
|
||||||
|
|
||||||
|
it('skips authored scenarios only while recording', () => {
|
||||||
|
expect(scenarioSkipped(authored, true, 'linux')).toBe(true)
|
||||||
|
expect(scenarioSkipped(authored, false, 'linux')).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('skips posixOnly scenarios on Windows and nowhere else', () => {
|
||||||
|
expect(scenarioSkipped(posix, false, 'win32')).toBe(true)
|
||||||
|
expect(scenarioSkipped(posix, false, 'linux')).toBe(false)
|
||||||
|
expect(scenarioSkipped(posix, false, 'darwin')).toBe(false)
|
||||||
|
expect(scenarioSkipped(authored, false, 'win32')).toBe(false)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe('fixtureContext', () => {
|
describe('fixtureContext', () => {
|
||||||
it('reads the fixture header id and cwd', () => {
|
it('reads the fixture header id and cwd', () => {
|
||||||
const ctx = fixtureContext('{"type":"session","id":"abc","cwd":"/rec"}\n{"type":"turn/start"}\n')
|
const ctx = fixtureContext('{"type":"session","id":"abc","cwd":"/rec"}\n{"type":"turn/start"}\n')
|
||||||
|
|||||||
Reference in New Issue
Block a user