test(acp): pin Windows-native snapshot paths

This commit is contained in:
imccyu
2026-07-17 13:53:13 +08:00
parent 86c09f6ca9
commit beb13c3808
10 changed files with 367 additions and 23 deletions
+10 -1
View File
@@ -150,6 +150,15 @@ export interface RunOptions {
configPath?: string
}
/**
* Return a fixed-length spill root across POSIX and Windows after Windows adds its drive prefix.
* @param platform - the host platform, injectable for unit coverage.
* @returns the root-relative snapshot spill directory.
*/
export function snapshotSpillRoot(platform: NodeJS.Platform = process.platform): string {
return platform === 'win32' ? '/t/dsh-acp-snapshot-spill' : '/tmp/dsh-acp-snapshot-spill'
}
/**
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
* child and its temp dirs; always tears them down. Returns the captured stdout
@@ -164,7 +173,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
// Fixed path length: spill-policy budgets the preview against the REAL path
// before stdout normalization, so tmpdir() length differences churn goldens.
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
const spillRoot = snapshotSpillRoot()
// Everything past the temp-dir creation runs under a try/finally that always
// removes both dirs — so a failure in workspace seeding, spawn, or any step
// never leaks them (the "e2e tests own their resources" rule).
@@ -21,7 +21,9 @@ export {
scrubRequestHeaders,
scrubSystemPrompts,
scrubToolSchemas,
type CwdPathMode,
type NormalizeContext,
type NormalizeOptions,
} from './normalize.ts'
export {
defineAcpSnapshotSuite,
+55 -11
View File
@@ -12,19 +12,33 @@ const SYSTEM = '{{system}}'
const TOOLS = '{{tools}}'
const MESSAGE_PREFIX = '{{messagePrefix}}'
/** A cwd-rooted path after volatile cwd replacement, through its last separator-delimited segment. */
const CWD_ROOTED_PATH_RE = /\{\{cwd\}\}(?:[\\/][^\s<>"'`]+)+/g
const PATH_TAG_RE = /(<path>)([^<]*)(<\/path>)/g
const ADDITIONAL_INSTRUCTIONS_PATH_RE = /(Additional instructions from: )([^\r\n]+)/g
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
const LOCAL_SPILL_PATH_RE = new RegExp(
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
String.raw`\{\{cwd\}\}[\\/]\.spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
String.raw`(?:[A-Za-z]:)?[\\/](?:tmp|t)[\\/]dsh-acp-snapshot-spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
/** Convert separators only inside generated path-bearing text markers. */
function canonicalizeEmbeddedPaths(value: string): string {
return value
.replace(PATH_TAG_RE, (_match, open: string, path: string, close: string) =>
`${open}${path.replaceAll('\\', '/')}${close}`)
.replace(ADDITIONAL_INSTRUCTIONS_PATH_RE, (_match, prefix: string, path: string) =>
`${prefix}${path.replaceAll('\\', '/')}`)
}
/** Inputs the normalizers need to recognize a run's volatile values. */
export interface NormalizeContext {
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
@@ -33,13 +47,28 @@ export interface NormalizeContext {
cwd: string
}
/** How cwd-rooted path separators are represented after the cwd is tokenized. */
export type CwdPathMode = 'canonical' | 'native'
/** Optional controls shared by stdout and session-log normalization. */
export interface NormalizeOptions {
/** Use `/` for shared goldens, or preserve captured separators for a platform-specific golden. */
cwdPathMode?: CwdPathMode
}
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
function scrubString(value: string, ctx: NormalizeContext): string {
function scrubString(value: string, ctx: NormalizeContext, cwdPathMode: CwdPathMode): string {
let out = value
// cwd first (longest, most specific), then explicit session ids, then any
// residual UUID (covers ids that appear in places we didn't enumerate).
out = out.split(ctx.cwd).join(CWD)
out = out.split(`/private${CWD}`).join(CWD)
if (cwdPathMode === 'canonical') {
// Restrict separator conversion to paths rooted at the cwd token. A global
// backslash rewrite would corrupt regexes, commands, and model-authored text.
out = out.replace(CWD_ROOTED_PATH_RE, path => path.replaceAll('\\', '/'))
out = canonicalizeEmbeddedPaths(out)
}
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
@@ -48,12 +77,15 @@ function scrubString(value: string, ctx: NormalizeContext): string {
}
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
if (typeof value === 'string') return scrubString(value, ctx)
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
function scrubValue(value: unknown, ctx: NormalizeContext, cwdPathMode: CwdPathMode, key?: string): unknown {
if (typeof value === 'string') {
const scrubbed = scrubString(value, ctx, cwdPathMode)
return cwdPathMode === 'canonical' && key === 'path' ? scrubbed.replaceAll('\\', '/') : scrubbed
}
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx, cwdPathMode))
if (value !== null && typeof value === 'object') {
const out: Record<string, unknown> = {}
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx, cwdPathMode, k)
return out
}
return value
@@ -67,9 +99,15 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
*
* @param rawStdout The captured stdout bytes, decoded utf8.
* @param ctx The run's volatile values to scrub.
* @param options Separator output controls; shared canonical paths are the default.
* @returns The normalized NDJSON transcript, one frame per line.
*/
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
export function normalizeStdout(
rawStdout: string,
ctx: NormalizeContext,
options: NormalizeOptions = {},
): string {
const cwdPathMode = options.cwdPathMode ?? 'canonical'
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
// sequence number, in first-seen order, so id churn doesn't perturb the golden.
@@ -85,7 +123,7 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
frame.id = stableId(frame.id)
}
return scrubValue(frame, ctx) as Record<string, unknown>
return scrubValue(frame, ctx, cwdPathMode) as Record<string, unknown>
})
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
}
@@ -99,9 +137,15 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
*
* @param rawLog The raw session `.jsonl` content.
* @param ctx The run's volatile values to scrub.
* @param options Separator output controls; shared canonical paths are the default.
* @returns The normalized JSONL log, one record per line.
*/
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
export function normalizeSessionLog(
rawLog: string,
ctx: NormalizeContext,
options: NormalizeOptions = {},
): string {
const cwdPathMode = options.cwdPathMode ?? 'canonical'
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
const records = lines.map((line) => {
const record = JSON.parse(line) as Record<string, unknown>
@@ -119,7 +163,7 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
if ('durationMs' in data) data.durationMs = 0
}
}
return scrubValue(record, ctx) as Record<string, unknown>
return scrubValue(record, ctx, cwdPathMode) as Record<string, unknown>
})
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
}
+44 -5
View File
@@ -21,6 +21,7 @@ import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
import {
type CwdPathMode,
type NormalizeContext,
normalizeSessionLog,
normalizeStdout,
@@ -35,6 +36,9 @@ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.golden.md'
/** The structured tool-schema snapshot beside each header-pinning fixture. */
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.golden.json'
/** The optional full Windows-native stdout transcript. */
const WINDOWS_STDOUT_SNAPSHOT = 'stdout.golden.windows.jsonl'
/** Stable session-log token standing in for the sidecar's initial schemas. */
const TOOLS_TOKEN = '{{tools}}'
@@ -108,6 +112,35 @@ export interface Scenario {
* {@link headerClass}.
*/
configPath?: string
/**
* Whether Windows additionally compares stdout with native separators against
* `stdout.golden.windows.jsonl`. The shared canonical stdout golden is still
* compared on every platform, and the fixture guard requires this sidecar
* exactly when the option is set.
*/
pinsNativeWindowsStdout?: boolean
}
/** One stdout golden selected for a platform run. */
interface StdoutGoldenVariant {
file: string
cwdPathMode: CwdPathMode
}
/**
* Select the shared stdout golden plus any platform-native assertion declared by a scenario.
*
* @param scenario The scenario whose stdout contract is being selected.
* @param platform The running Node platform, injectable for unit coverage.
* @returns The ordered golden variants: shared canonical first, then optional Windows native.
*/
export function stdoutGoldenVariants(
scenario: Scenario,
platform: NodeJS.Platform = process.platform,
): StdoutGoldenVariant[] {
const canonical: StdoutGoldenVariant = { file: 'stdout.golden.jsonl', cwdPathMode: 'canonical' }
if (platform !== 'win32' || scenario.pinsNativeWindowsStdout !== true) return [canonical]
return [canonical, { file: WINDOWS_STDOUT_SNAPSHOT, cwdPathMode: 'native' }]
}
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
@@ -530,11 +563,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
const stdout = normalizeStdout(result.rawStdout, ctx)
if (REFRESHING) {
await writeFile(join(dir, 'stdout.golden.jsonl'), stdout)
for (const golden of stdoutGoldenVariants(scenario)) {
const stdout = normalizeStdout(result.rawStdout, ctx, { cwdPathMode: golden.cwdPathMode })
if (REFRESHING) {
await writeFile(join(dir, golden.file), stdout)
}
await expect(stdout, `${golden.file} mismatch`).toMatchFileSnapshot(join(dir, golden.file))
}
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
// A model turn always produces a log worth comparing; a hook scenario can
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
@@ -621,10 +656,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
it('every registered scenario has its required fixture files', () => {
// Every scenario has an input script and an stdout golden.
for (const { name, overridden, childSessions, pinsHeader } of scenarios) {
for (const { name, overridden, childSessions, pinsHeader, pinsNativeWindowsStdout } of scenarios) {
const dir = join(snapshotsDir, name)
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
expect(
existsSync(join(dir, WINDOWS_STDOUT_SNAPSHOT)),
`${name}/${WINDOWS_STDOUT_SNAPSHOT} presence must match \`pinsNativeWindowsStdout\``,
).toBe(pinsNativeWindowsStdout === true)
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
.toBe(overridden === true)