Merge remote-tracking branch 'origin/master' into codex/session-scoped-sandbox-roots
# Conflicts: # .agents/notes/implemented/feature/2026-07-14-cross-family-fs-sandbox.i18n.yaml # .agents/notes/implemented/feature/2026-07-14-cross-family-fs-sandbox.md # .agents/notes/implemented/feature/2026-07-14-cross-family-fs-sandbox.zh.md # examples/acp-agent/README.md # packages/examples/agent-spine-demo/package.json # packages/fs/fs-sandbox/src/index.ts # packages/fs/tool-fs-search/tests/tools.spec.ts # packages/support/acp-snapshot/README.md # packages/support/acp-snapshot/src/suite.ts # pnpm-lock.yaml # scripts/type-equiv.manifest.json
This commit is contained in:
@@ -24,6 +24,8 @@ import { basename, dirname, join, delimiter } from 'node:path'
|
||||
import {
|
||||
ClientSideConnection,
|
||||
PROTOCOL_VERSION,
|
||||
type CreateElicitationRequest,
|
||||
type CreateElicitationResponse,
|
||||
type RequestPermissionRequest,
|
||||
type RequestPermissionResponse,
|
||||
type SessionNotification,
|
||||
@@ -59,6 +61,8 @@ export type InputStep =
|
||||
waitForToolCallUpdate?: string
|
||||
}
|
||||
| { op: 'cancel' }
|
||||
| { op: 'setMode'; modeId: string }
|
||||
| { op: 'setModeExpectError'; modeId: string }
|
||||
| { op: 'setConfigOption'; configId: string; value: string }
|
||||
| { op: 'setConfigOptionExpectError'; configId: string; value: string }
|
||||
|
||||
@@ -78,6 +82,16 @@ export interface InputScript {
|
||||
* agent itself just sees `cancelled`, so it cannot absorb the bug).
|
||||
*/
|
||||
permissionAnswers?: PermissionAnswer[]
|
||||
/**
|
||||
* Ordered answers for the agent's `elicitation/create` round-trips (the
|
||||
* ask_user_question / plan-review forms), consumed FIFO — the Nth request
|
||||
* gets the Nth answer. Exhaustion (or no queue) answers `cancel`, the same
|
||||
* fail-closed stub an elicitation-free scenario relies on. Unlike permission
|
||||
* kinds, the scripted strings are not validated against the offered form —
|
||||
* a stray `choice` reaches the agent verbatim, which reads it as a custom
|
||||
* (non-consenting) answer, so a scenario bug fails safe in the transcript.
|
||||
*/
|
||||
elicitationAnswers?: ElicitationAnswer[]
|
||||
}
|
||||
|
||||
/** One scripted answer to a permission request: which offered option kind to select. */
|
||||
@@ -86,6 +100,16 @@ export interface PermissionAnswer {
|
||||
kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
|
||||
}
|
||||
|
||||
/** One scripted answer to an elicitation form (accept with choice/custom content, or cancel). */
|
||||
export interface ElicitationAnswer {
|
||||
/** Accept the form with the content below, or cancel it. */
|
||||
action: 'accept' | 'cancel'
|
||||
/** The selected option label (the form's `choice` field). */
|
||||
choice?: string
|
||||
/** Free-form text (the form's `custom` field). */
|
||||
custom?: string
|
||||
}
|
||||
|
||||
/** One harvested session log plus the identifying facts off its header line. */
|
||||
export interface HarvestedLog {
|
||||
/** The recorded session id (header `id`). */
|
||||
@@ -161,11 +185,21 @@ export interface RunOptions {
|
||||
configPath?: string
|
||||
}
|
||||
|
||||
/** Derive one stable, fixed-length spill root owned by this scenario. */
|
||||
function scenarioSpillRoot(fixtureFile: string): string {
|
||||
/**
|
||||
* Derive one stable, fixed-length spill root owned by this scenario.
|
||||
* Windows uses a two-character-shorter root because drive resolution adds its drive prefix.
|
||||
* @param fixtureFile - The scenario fixture whose parent directory provides the stable identity.
|
||||
* @param platform - the host platform, injectable for unit coverage.
|
||||
* @returns the root-relative snapshot spill directory.
|
||||
*/
|
||||
export function snapshotSpillRoot(
|
||||
fixtureFile: string,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): string {
|
||||
const scenario = basename(dirname(fixtureFile))
|
||||
const key = createHash('sha256').update(scenario).digest('hex').slice(0, 9)
|
||||
return `/tmp/dsh-acp-snap-${key}`
|
||||
const root = platform === 'win32' ? '/t' : '/tmp'
|
||||
return `${root}/dsh-acp-snap-${key}`
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -184,7 +218,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
// before stdout normalization, so tmpdir() length differences churn expected outputs.
|
||||
// Scenario ownership also matters: replay runs concurrently, and one teardown
|
||||
// must never delete another scenario's in-flight full-output recovery file.
|
||||
const spillRoot = scenarioSpillRoot(opts.fixtureFile)
|
||||
const spillRoot = snapshotSpillRoot(opts.fixtureFile)
|
||||
// Everything past the temp-dir creation is followed by failure-safe cleanup,
|
||||
// so a failure in workspace seeding, spawn, or any step never leaks resources.
|
||||
let launched: LaunchedAcpTestAgent | undefined
|
||||
@@ -213,6 +247,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
// Permission answers are consumed FIFO across the whole run; exhaustion
|
||||
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
|
||||
const permissionQueue = [...input.permissionAnswers ?? []]
|
||||
// Elicitation answers mirror the permission queue: FIFO, cancel on exhaustion.
|
||||
const elicitationQueue = [...input.elicitationAnswers ?? []]
|
||||
// A scenario bug detected inside a client callback (a scripted permission
|
||||
// kind the agent never offered). It cannot fail the run from in there: a
|
||||
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
|
||||
@@ -242,6 +278,17 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
}
|
||||
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
|
||||
},
|
||||
createElicitation(_params: CreateElicitationRequest): Promise<CreateElicitationResponse> {
|
||||
const answer = elicitationQueue.shift()
|
||||
if (answer === undefined || answer.action !== 'accept') return Promise.resolve({ action: 'cancel' })
|
||||
return Promise.resolve({
|
||||
action: 'accept',
|
||||
content: {
|
||||
...answer.choice !== undefined ? { choice: answer.choice } : {},
|
||||
...answer.custom !== undefined ? { custom: answer.custom } : {},
|
||||
},
|
||||
})
|
||||
},
|
||||
})
|
||||
const active = launched
|
||||
await active.spawned
|
||||
@@ -397,6 +444,24 @@ async function runStep(
|
||||
await client.cancel({ sessionId })
|
||||
return
|
||||
}
|
||||
case 'setMode': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setMode before newSession')
|
||||
await client.setSessionMode({ sessionId, modeId: step.modeId })
|
||||
return
|
||||
}
|
||||
case 'setModeExpectError': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setModeExpectError before newSession')
|
||||
// The bridge rejects an unknown/uncomposed mode id with invalidParams;
|
||||
// that rejection IS the expected wire behavior — swallow it so the run
|
||||
// completes and the error frame is captured in the transcript.
|
||||
await client.setSessionMode({ sessionId, modeId: step.modeId }).then(
|
||||
() => { throw new Error('snapshot-harness: expected session/set_mode to be rejected but it succeeded') },
|
||||
() => { /* expected: the bridge rejected the mode id */ },
|
||||
)
|
||||
return
|
||||
}
|
||||
case 'setConfigOption': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
export {
|
||||
runScenario,
|
||||
type ElicitationAnswer,
|
||||
type HarvestedLog,
|
||||
type InputScript,
|
||||
type InputStep,
|
||||
@@ -37,10 +38,14 @@ export {
|
||||
scrubRequestHeaders,
|
||||
scrubSystemPrompts,
|
||||
scrubToolSchemas,
|
||||
type CwdPathMode,
|
||||
type NormalizeContext,
|
||||
type NormalizeOptions,
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
defineAcpSnapshotSuite,
|
||||
refreshFixtureReplacements,
|
||||
stabilizeRefreshLog,
|
||||
type Scenario,
|
||||
type SnapshotSuiteOptions,
|
||||
} from './suite.ts'
|
||||
|
||||
@@ -15,12 +15,16 @@ import {
|
||||
ndJsonStream,
|
||||
type Agent as AcpAgent,
|
||||
type Client,
|
||||
type CreateElicitationRequest,
|
||||
type CreateElicitationResponse,
|
||||
type RequestPermissionRequest,
|
||||
type RequestPermissionResponse,
|
||||
type SessionNotification,
|
||||
} from '@agentclientprotocol/sdk'
|
||||
import { resolveExampleLaunch } from '@deepseek-ai/dsh-loader-smoke'
|
||||
|
||||
const EXIT_MARKER_GRACE_MS = 250
|
||||
|
||||
/** The source/built agent entry, leaf config, and workspace tsconfig an ACP test boots. */
|
||||
export interface AgentUnderTest {
|
||||
/** The agent source bin entry (for example `packages/examples/acp-demo/src/bin.ts`). */
|
||||
@@ -45,6 +49,8 @@ export interface AcpTestLaunchOptions {
|
||||
env?: NodeJS.ProcessEnv
|
||||
/** Permission handler; omitted requests fail closed as `cancelled`. */
|
||||
requestPermission?: (params: RequestPermissionRequest) => Promise<RequestPermissionResponse>
|
||||
/** Elicitation handler; omitted requests fail closed as `cancel`. */
|
||||
createElicitation?: (params: CreateElicitationRequest) => Promise<CreateElicitationResponse>
|
||||
}
|
||||
|
||||
/** A running ACP test process and its captured client-side surfaces. */
|
||||
@@ -150,6 +156,8 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
}
|
||||
const requestPermission = options.requestPermission
|
||||
?? (() => Promise.resolve({ outcome: { outcome: 'cancelled' as const } }))
|
||||
const createElicitation = options.createElicitation
|
||||
?? (() => Promise.resolve({ action: 'cancel' as const }))
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(params: SessionNotification): Promise<void> {
|
||||
return trackClientCallback(() => {
|
||||
@@ -173,6 +181,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
})
|
||||
},
|
||||
requestPermission: params => trackClientCallback(() => requestPermission(params)),
|
||||
unstable_createElicitation: params => trackClientCallback(() => createElicitation(params)),
|
||||
})
|
||||
const client = new ClientSideConnection(makeClient, stream)
|
||||
// `exit` only reports the parent process's status. Descendants may retain
|
||||
@@ -231,6 +240,15 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
return
|
||||
}
|
||||
|
||||
const propagateFailureAfterDrain = async (): Promise<never> => {
|
||||
await drained
|
||||
closeUpdateStream()
|
||||
throw failure
|
||||
}
|
||||
// Windows implements the supported signal names as forced termination. The exit markers
|
||||
// may therefore arrive after the error wins the race above but before fallback begins.
|
||||
if (!isRunning(child) || await exitMarkerWithinGrace(exited)) return propagateFailureAfterDrain()
|
||||
|
||||
// An `error` after spawn is not an exit edge: in particular, a failed
|
||||
// signal can leave the subprocess live. Force termination, await the
|
||||
// already-observed exit edge, and only then propagate the child error so
|
||||
@@ -240,6 +258,10 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
child.once('error', observeFallbackError)
|
||||
if (!child.kill('SIGKILL')) {
|
||||
child.off('error', observeFallbackError)
|
||||
// A successful earlier signal may win between the live check and this fallback call.
|
||||
// In that case `kill()` correctly reports no process to signal; the original child error
|
||||
// remains the shutdown result once inherited stdio and callbacks have drained.
|
||||
if (!isRunning(child) || await exitMarkerWithinGrace(exited)) return propagateFailureAfterDrain()
|
||||
closeUpdateStream()
|
||||
throw new AggregateError(
|
||||
[failure, new Error('Fallback SIGKILL was not accepted by the child process')],
|
||||
@@ -258,9 +280,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
'ACP test agent failed and fallback termination was refused',
|
||||
)
|
||||
}
|
||||
await drained
|
||||
closeUpdateStream()
|
||||
throw failure
|
||||
return propagateFailureAfterDrain()
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -270,6 +290,17 @@ function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
|
||||
}
|
||||
|
||||
/** Give an accepted Windows termination request a bounded window to publish its exit marker. */
|
||||
function exitMarkerWithinGrace(exited: Promise<void>): Promise<boolean> {
|
||||
return Promise.race([
|
||||
exited.then(() => true),
|
||||
new Promise<false>((resolve) => {
|
||||
const timer = setTimeout(() => { resolve(false) }, EXIT_MARKER_GRACE_MS)
|
||||
timer.unref()
|
||||
}),
|
||||
])
|
||||
}
|
||||
|
||||
/** Whether the child still lacks either OS termination marker. */
|
||||
function isRunning(child: ChildProcessWithoutNullStreams): boolean {
|
||||
return child.exitCode === null && child.signalCode === null
|
||||
|
||||
@@ -11,20 +11,35 @@ const CWD = '{{cwd}}'
|
||||
const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
const MESSAGE_PREFIX = '{{messagePrefix}}'
|
||||
const UPDATED_AT = '{{updatedAt}}'
|
||||
|
||||
/** A cwd-rooted path after volatile cwd replacement, through its last separator-delimited segment. */
|
||||
const CWD_ROOTED_PATH_RE = /\{\{cwd\}\}(?:[\\/][^\s<>"'`]+)+/g
|
||||
const PATH_TAG_RE = /(<path>)([^<]*)(<\/path>)/g
|
||||
const ADDITIONAL_INSTRUCTIONS_PATH_RE = /(Additional instructions from: )([^\r\n]+)/g
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
const LOCAL_SPILL_PATH_RE = new RegExp(
|
||||
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
String.raw`\{\{cwd\}\}[\\/]\.spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||
'g',
|
||||
)
|
||||
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
|
||||
String.raw`/tmp/(?:dsh-acp-snap-[0-9a-f]{9}|dsh-acp-snapshot-spill)/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
String.raw`(?:[A-Za-z]:)?[\\/](?:tmp|t)[\\/](?:dsh-acp-snap-[0-9a-f]{9}|dsh-acp-snapshot-spill)[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||
'g',
|
||||
)
|
||||
|
||||
/** Convert separators only inside generated path-bearing text markers. */
|
||||
function canonicalizeEmbeddedPaths(value: string): string {
|
||||
return value
|
||||
.replace(PATH_TAG_RE, (_match, open: string, path: string, close: string) =>
|
||||
`${open}${path.replaceAll('\\', '/')}${close}`)
|
||||
.replace(ADDITIONAL_INSTRUCTIONS_PATH_RE, (_match, prefix: string, path: string) =>
|
||||
`${prefix}${path.replaceAll('\\', '/')}`)
|
||||
}
|
||||
|
||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||
export interface NormalizeContext {
|
||||
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
|
||||
@@ -33,13 +48,28 @@ export interface NormalizeContext {
|
||||
cwd: string
|
||||
}
|
||||
|
||||
/** How cwd-rooted path separators are represented after the cwd is tokenized. */
|
||||
export type CwdPathMode = 'canonical' | 'native'
|
||||
|
||||
/** Optional controls shared by stdout and session-log normalization. */
|
||||
export interface NormalizeOptions {
|
||||
/** Use `/` for shared goldens, or preserve captured separators for a platform-specific golden. */
|
||||
cwdPathMode?: CwdPathMode
|
||||
}
|
||||
|
||||
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
|
||||
function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
function scrubString(value: string, ctx: NormalizeContext, cwdPathMode: CwdPathMode): string {
|
||||
let out = value
|
||||
// cwd first (longest, most specific), then explicit session ids, then any
|
||||
// residual UUID (covers ids that appear in places we didn't enumerate).
|
||||
out = out.split(ctx.cwd).join(CWD)
|
||||
out = out.split(`/private${CWD}`).join(CWD)
|
||||
if (cwdPathMode === 'canonical') {
|
||||
// Restrict separator conversion to paths rooted at the cwd token. A global
|
||||
// backslash rewrite would corrupt regexes, commands, and model-authored text.
|
||||
out = out.replace(CWD_ROOTED_PATH_RE, path => path.replaceAll('\\', '/'))
|
||||
out = canonicalizeEmbeddedPaths(out)
|
||||
}
|
||||
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
@@ -48,12 +78,15 @@ function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
}
|
||||
|
||||
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
if (typeof value === 'string') return scrubString(value, ctx)
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext, cwdPathMode: CwdPathMode, key?: string): unknown {
|
||||
if (typeof value === 'string') {
|
||||
const scrubbed = scrubString(value, ctx, cwdPathMode)
|
||||
return cwdPathMode === 'canonical' && key === 'path' ? scrubbed.replaceAll('\\', '/') : scrubbed
|
||||
}
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx, cwdPathMode))
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx, cwdPathMode, k)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
@@ -67,9 +100,15 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
*
|
||||
* @param rawStdout The captured stdout bytes, decoded utf8.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @param options Separator output controls; shared canonical paths are the default.
|
||||
* @returns The normalized NDJSON transcript, one frame per line.
|
||||
*/
|
||||
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
|
||||
export function normalizeStdout(
|
||||
rawStdout: string,
|
||||
ctx: NormalizeContext,
|
||||
options: NormalizeOptions = {},
|
||||
): string {
|
||||
const cwdPathMode = options.cwdPathMode ?? 'canonical'
|
||||
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
|
||||
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
|
||||
// sequence number, in first-seen order, so id churn doesn't perturb the expected output.
|
||||
@@ -85,7 +124,9 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
|
||||
frame.id = stableId(frame.id)
|
||||
}
|
||||
return scrubValue(frame, ctx) as Record<string, unknown>
|
||||
const update = (frame.params as { update?: Record<string, unknown> } | undefined)?.update
|
||||
if (update?.sessionUpdate === 'session_info_update') update.updatedAt = UPDATED_AT
|
||||
return scrubValue(frame, ctx, cwdPathMode) as Record<string, unknown>
|
||||
})
|
||||
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
|
||||
}
|
||||
@@ -99,9 +140,15 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @param options Separator output controls; shared canonical paths are the default.
|
||||
* @returns The normalized JSONL log, one record per line.
|
||||
*/
|
||||
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
|
||||
export function normalizeSessionLog(
|
||||
rawLog: string,
|
||||
ctx: NormalizeContext,
|
||||
options: NormalizeOptions = {},
|
||||
): string {
|
||||
const cwdPathMode = options.cwdPathMode ?? 'canonical'
|
||||
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
|
||||
const records = lines.map((line) => {
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
@@ -119,7 +166,7 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
||||
if ('durationMs' in data) data.durationMs = 0
|
||||
}
|
||||
}
|
||||
return scrubValue(record, ctx) as Record<string, unknown>
|
||||
return scrubValue(record, ctx, cwdPathMode) as Record<string, unknown>
|
||||
})
|
||||
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
|
||||
import {
|
||||
type CwdPathMode,
|
||||
type NormalizeContext,
|
||||
normalizeSessionLog,
|
||||
normalizeStdout,
|
||||
@@ -35,6 +36,9 @@ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md'
|
||||
/** The structured tool-schema snapshot beside each header-pinning fixture. */
|
||||
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.expected.json'
|
||||
|
||||
/** The optional full Windows-native stdout transcript. */
|
||||
const WINDOWS_STDOUT_SNAPSHOT = 'stdout.expected.windows.jsonl'
|
||||
|
||||
/** Stable session-log token standing in for the sidecar's initial schemas. */
|
||||
const TOOLS_TOKEN = '{{tools}}'
|
||||
|
||||
@@ -106,6 +110,61 @@ export interface Scenario {
|
||||
* test and the scenario needs an independent project location.
|
||||
*/
|
||||
workspaceParent?: string
|
||||
/**
|
||||
* Whether Windows additionally compares stdout with native separators against
|
||||
* `stdout.expected.windows.jsonl`. The shared canonical stdout expected output is still
|
||||
* compared on every platform, and the fixture guard requires this sidecar
|
||||
* exactly when the option is set.
|
||||
*/
|
||||
pinsNativeWindowsStdout?: boolean
|
||||
/**
|
||||
* Whether the driven behavior needs POSIX process semantics the harness
|
||||
* cannot exercise on Windows (e.g. cancelling a live bash tool call kills a
|
||||
* detached process group). The scenario's run test is skipped on Windows;
|
||||
* its fixtures stay guarded on every platform.
|
||||
*/
|
||||
posixOnly?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a scenario's run test is skipped for this mode and host: record mode
|
||||
* skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly}
|
||||
* scenarios skip on Windows.
|
||||
*
|
||||
* @param scenario The scenario whose run test is being registered.
|
||||
* @param recording Whether the suite runs in record mode.
|
||||
* @param platform The running Node platform, injectable for unit coverage.
|
||||
* @returns True when the scenario's run test must not execute.
|
||||
*/
|
||||
export function scenarioSkipped(
|
||||
scenario: Scenario,
|
||||
recording: boolean,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): boolean {
|
||||
if (recording && !scenario.recorded) return true
|
||||
return scenario.posixOnly === true && platform === 'win32'
|
||||
}
|
||||
|
||||
/** One stdout expected output selected for a platform run. */
|
||||
interface StdoutExpectedVariant {
|
||||
file: string
|
||||
cwdPathMode: CwdPathMode
|
||||
}
|
||||
|
||||
/**
|
||||
* Select the shared stdout expected output plus any platform-native assertion declared by a scenario.
|
||||
*
|
||||
* @param scenario The scenario whose stdout contract is being selected.
|
||||
* @param platform The running Node platform, injectable for unit coverage.
|
||||
* @returns The ordered expected-output variants: shared canonical first, then optional Windows native.
|
||||
*/
|
||||
export function stdoutExpectedVariants(
|
||||
scenario: Scenario,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): StdoutExpectedVariant[] {
|
||||
const canonical: StdoutExpectedVariant = { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' }
|
||||
if (platform !== 'win32' || scenario.pinsNativeWindowsStdout !== true) return [canonical]
|
||||
return [canonical, { file: WINDOWS_STDOUT_SNAPSHOT, cwdPathMode: 'native' }]
|
||||
}
|
||||
|
||||
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
|
||||
@@ -426,8 +485,21 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
|
||||
for (const { from, to } of replacements) stable = stable.split(from).join(to)
|
||||
const existingRecords = parseJsonlRecords(existing)
|
||||
const records = parseJsonlRecords(stable)
|
||||
let existingIndex = 0
|
||||
let previousEventTime: unknown
|
||||
for (let i = 0; i < records.length; i++) {
|
||||
preserveFixtureVolatiles(records[i] as Record<string, unknown>, existingRecords[i])
|
||||
const record = records[i] as Record<string, unknown>
|
||||
const existingRecord = existingRecords[existingIndex]
|
||||
const insertedTitle = record.type === 'session/title' && existingRecord?.type !== 'session/title'
|
||||
if (insertedTitle) {
|
||||
/* v8 ignore next -- a title is turn-enclosed, so a preceding event time exists in every valid fixture. */
|
||||
if (typeof previousEventTime !== 'number') throw new Error('acp-snapshot: inserted title has no preceding event time')
|
||||
record.time = previousEventTime
|
||||
} else {
|
||||
preserveFixtureVolatiles(record, existingRecord)
|
||||
existingIndex += 1
|
||||
}
|
||||
if (typeof record.time === 'number') previousEventTime = record.time
|
||||
}
|
||||
return records.map(record => JSON.stringify(record)).join('\n') + '\n'
|
||||
}
|
||||
@@ -472,8 +544,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
scenarioSuite('snapshot scenarios', () => {
|
||||
for (const scenario of scenarios) {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||
// (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
|
||||
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on
|
||||
// Windows, where their process semantics cannot be driven.
|
||||
it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
@@ -578,11 +651,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
}
|
||||
|
||||
const stdout = normalizeStdout(result.rawStdout, ctx)
|
||||
if (REFRESHING) {
|
||||
await writeFile(join(dir, 'stdout.expected.jsonl'), stdout)
|
||||
for (const expected of stdoutExpectedVariants(scenario)) {
|
||||
const stdout = normalizeStdout(result.rawStdout, ctx, { cwdPathMode: expected.cwdPathMode })
|
||||
if (REFRESHING) {
|
||||
await writeFile(join(dir, expected.file), stdout)
|
||||
}
|
||||
await expect(stdout, `${expected.file} mismatch`).toMatchFileSnapshot(join(dir, expected.file))
|
||||
}
|
||||
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.expected.jsonl'))
|
||||
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
@@ -669,10 +744,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
|
||||
it('every registered scenario has its required fixture files', async () => {
|
||||
// Every scenario needs input, stdout, a primary session fixture, and matching optional sidecars.
|
||||
for (const { name, overridden, pinsHeader } of scenarios) {
|
||||
for (const { name, overridden, pinsHeader, pinsNativeWindowsStdout } of scenarios) {
|
||||
const dir = join(snapshotsDir, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.expected.jsonl')), `${name}/stdout.expected.jsonl`).toBe(true)
|
||||
expect(
|
||||
existsSync(join(dir, WINDOWS_STDOUT_SNAPSHOT)),
|
||||
`${name}/${WINDOWS_STDOUT_SNAPSHOT} presence must match \`pinsNativeWindowsStdout\``,
|
||||
).toBe(pinsNativeWindowsStdout === true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
|
||||
.toBe(overridden === true)
|
||||
|
||||
Reference in New Issue
Block a user