test(acp): pin Windows-native snapshot paths
This commit is contained in:
@@ -150,6 +150,15 @@ export interface RunOptions {
|
||||
configPath?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a fixed-length spill root across POSIX and Windows after Windows adds its drive prefix.
|
||||
* @param platform - the host platform, injectable for unit coverage.
|
||||
* @returns the root-relative snapshot spill directory.
|
||||
*/
|
||||
export function snapshotSpillRoot(platform: NodeJS.Platform = process.platform): string {
|
||||
return platform === 'win32' ? '/t/dsh-acp-snapshot-spill' : '/tmp/dsh-acp-snapshot-spill'
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
|
||||
* child and its temp dirs; always tears them down. Returns the captured stdout
|
||||
@@ -164,7 +173,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
||||
// Fixed path length: spill-policy budgets the preview against the REAL path
|
||||
// before stdout normalization, so tmpdir() length differences churn goldens.
|
||||
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
|
||||
const spillRoot = snapshotSpillRoot()
|
||||
// Everything past the temp-dir creation runs under a try/finally that always
|
||||
// removes both dirs — so a failure in workspace seeding, spawn, or any step
|
||||
// never leaks them (the "e2e tests own their resources" rule).
|
||||
|
||||
@@ -21,7 +21,9 @@ export {
|
||||
scrubRequestHeaders,
|
||||
scrubSystemPrompts,
|
||||
scrubToolSchemas,
|
||||
type CwdPathMode,
|
||||
type NormalizeContext,
|
||||
type NormalizeOptions,
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
defineAcpSnapshotSuite,
|
||||
|
||||
@@ -12,19 +12,33 @@ const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
const MESSAGE_PREFIX = '{{messagePrefix}}'
|
||||
|
||||
/** A cwd-rooted path after volatile cwd replacement, through its last separator-delimited segment. */
|
||||
const CWD_ROOTED_PATH_RE = /\{\{cwd\}\}(?:[\\/][^\s<>"'`]+)+/g
|
||||
const PATH_TAG_RE = /(<path>)([^<]*)(<\/path>)/g
|
||||
const ADDITIONAL_INSTRUCTIONS_PATH_RE = /(Additional instructions from: )([^\r\n]+)/g
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
const LOCAL_SPILL_PATH_RE = new RegExp(
|
||||
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
String.raw`\{\{cwd\}\}[\\/]\.spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||
'g',
|
||||
)
|
||||
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
|
||||
String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
String.raw`(?:[A-Za-z]:)?[\\/](?:tmp|t)[\\/]dsh-acp-snapshot-spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||
'g',
|
||||
)
|
||||
|
||||
/** Convert separators only inside generated path-bearing text markers. */
|
||||
function canonicalizeEmbeddedPaths(value: string): string {
|
||||
return value
|
||||
.replace(PATH_TAG_RE, (_match, open: string, path: string, close: string) =>
|
||||
`${open}${path.replaceAll('\\', '/')}${close}`)
|
||||
.replace(ADDITIONAL_INSTRUCTIONS_PATH_RE, (_match, prefix: string, path: string) =>
|
||||
`${prefix}${path.replaceAll('\\', '/')}`)
|
||||
}
|
||||
|
||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||
export interface NormalizeContext {
|
||||
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
|
||||
@@ -33,13 +47,28 @@ export interface NormalizeContext {
|
||||
cwd: string
|
||||
}
|
||||
|
||||
/** How cwd-rooted path separators are represented after the cwd is tokenized. */
|
||||
export type CwdPathMode = 'canonical' | 'native'
|
||||
|
||||
/** Optional controls shared by stdout and session-log normalization. */
|
||||
export interface NormalizeOptions {
|
||||
/** Use `/` for shared goldens, or preserve captured separators for a platform-specific golden. */
|
||||
cwdPathMode?: CwdPathMode
|
||||
}
|
||||
|
||||
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
|
||||
function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
function scrubString(value: string, ctx: NormalizeContext, cwdPathMode: CwdPathMode): string {
|
||||
let out = value
|
||||
// cwd first (longest, most specific), then explicit session ids, then any
|
||||
// residual UUID (covers ids that appear in places we didn't enumerate).
|
||||
out = out.split(ctx.cwd).join(CWD)
|
||||
out = out.split(`/private${CWD}`).join(CWD)
|
||||
if (cwdPathMode === 'canonical') {
|
||||
// Restrict separator conversion to paths rooted at the cwd token. A global
|
||||
// backslash rewrite would corrupt regexes, commands, and model-authored text.
|
||||
out = out.replace(CWD_ROOTED_PATH_RE, path => path.replaceAll('\\', '/'))
|
||||
out = canonicalizeEmbeddedPaths(out)
|
||||
}
|
||||
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
@@ -48,12 +77,15 @@ function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
}
|
||||
|
||||
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
if (typeof value === 'string') return scrubString(value, ctx)
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext, cwdPathMode: CwdPathMode, key?: string): unknown {
|
||||
if (typeof value === 'string') {
|
||||
const scrubbed = scrubString(value, ctx, cwdPathMode)
|
||||
return cwdPathMode === 'canonical' && key === 'path' ? scrubbed.replaceAll('\\', '/') : scrubbed
|
||||
}
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx, cwdPathMode))
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx, cwdPathMode, k)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
@@ -67,9 +99,15 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
*
|
||||
* @param rawStdout The captured stdout bytes, decoded utf8.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @param options Separator output controls; shared canonical paths are the default.
|
||||
* @returns The normalized NDJSON transcript, one frame per line.
|
||||
*/
|
||||
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
|
||||
export function normalizeStdout(
|
||||
rawStdout: string,
|
||||
ctx: NormalizeContext,
|
||||
options: NormalizeOptions = {},
|
||||
): string {
|
||||
const cwdPathMode = options.cwdPathMode ?? 'canonical'
|
||||
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
|
||||
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
|
||||
// sequence number, in first-seen order, so id churn doesn't perturb the golden.
|
||||
@@ -85,7 +123,7 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
|
||||
frame.id = stableId(frame.id)
|
||||
}
|
||||
return scrubValue(frame, ctx) as Record<string, unknown>
|
||||
return scrubValue(frame, ctx, cwdPathMode) as Record<string, unknown>
|
||||
})
|
||||
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
|
||||
}
|
||||
@@ -99,9 +137,15 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @param options Separator output controls; shared canonical paths are the default.
|
||||
* @returns The normalized JSONL log, one record per line.
|
||||
*/
|
||||
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
|
||||
export function normalizeSessionLog(
|
||||
rawLog: string,
|
||||
ctx: NormalizeContext,
|
||||
options: NormalizeOptions = {},
|
||||
): string {
|
||||
const cwdPathMode = options.cwdPathMode ?? 'canonical'
|
||||
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
|
||||
const records = lines.map((line) => {
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
@@ -119,7 +163,7 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
||||
if ('durationMs' in data) data.durationMs = 0
|
||||
}
|
||||
}
|
||||
return scrubValue(record, ctx) as Record<string, unknown>
|
||||
return scrubValue(record, ctx, cwdPathMode) as Record<string, unknown>
|
||||
})
|
||||
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
|
||||
import {
|
||||
type CwdPathMode,
|
||||
type NormalizeContext,
|
||||
normalizeSessionLog,
|
||||
normalizeStdout,
|
||||
@@ -35,6 +36,9 @@ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.golden.md'
|
||||
/** The structured tool-schema snapshot beside each header-pinning fixture. */
|
||||
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.golden.json'
|
||||
|
||||
/** The optional full Windows-native stdout transcript. */
|
||||
const WINDOWS_STDOUT_SNAPSHOT = 'stdout.golden.windows.jsonl'
|
||||
|
||||
/** Stable session-log token standing in for the sidecar's initial schemas. */
|
||||
const TOOLS_TOKEN = '{{tools}}'
|
||||
|
||||
@@ -108,6 +112,35 @@ export interface Scenario {
|
||||
* {@link headerClass}.
|
||||
*/
|
||||
configPath?: string
|
||||
/**
|
||||
* Whether Windows additionally compares stdout with native separators against
|
||||
* `stdout.golden.windows.jsonl`. The shared canonical stdout golden is still
|
||||
* compared on every platform, and the fixture guard requires this sidecar
|
||||
* exactly when the option is set.
|
||||
*/
|
||||
pinsNativeWindowsStdout?: boolean
|
||||
}
|
||||
|
||||
/** One stdout golden selected for a platform run. */
|
||||
interface StdoutGoldenVariant {
|
||||
file: string
|
||||
cwdPathMode: CwdPathMode
|
||||
}
|
||||
|
||||
/**
|
||||
* Select the shared stdout golden plus any platform-native assertion declared by a scenario.
|
||||
*
|
||||
* @param scenario The scenario whose stdout contract is being selected.
|
||||
* @param platform The running Node platform, injectable for unit coverage.
|
||||
* @returns The ordered golden variants: shared canonical first, then optional Windows native.
|
||||
*/
|
||||
export function stdoutGoldenVariants(
|
||||
scenario: Scenario,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): StdoutGoldenVariant[] {
|
||||
const canonical: StdoutGoldenVariant = { file: 'stdout.golden.jsonl', cwdPathMode: 'canonical' }
|
||||
if (platform !== 'win32' || scenario.pinsNativeWindowsStdout !== true) return [canonical]
|
||||
return [canonical, { file: WINDOWS_STDOUT_SNAPSHOT, cwdPathMode: 'native' }]
|
||||
}
|
||||
|
||||
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
|
||||
@@ -530,11 +563,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
}
|
||||
|
||||
const stdout = normalizeStdout(result.rawStdout, ctx)
|
||||
if (REFRESHING) {
|
||||
await writeFile(join(dir, 'stdout.golden.jsonl'), stdout)
|
||||
for (const golden of stdoutGoldenVariants(scenario)) {
|
||||
const stdout = normalizeStdout(result.rawStdout, ctx, { cwdPathMode: golden.cwdPathMode })
|
||||
if (REFRESHING) {
|
||||
await writeFile(join(dir, golden.file), stdout)
|
||||
}
|
||||
await expect(stdout, `${golden.file} mismatch`).toMatchFileSnapshot(join(dir, golden.file))
|
||||
}
|
||||
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
|
||||
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
@@ -621,10 +656,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
|
||||
it('every registered scenario has its required fixture files', () => {
|
||||
// Every scenario has an input script and an stdout golden.
|
||||
for (const { name, overridden, childSessions, pinsHeader } of scenarios) {
|
||||
for (const { name, overridden, childSessions, pinsHeader, pinsNativeWindowsStdout } of scenarios) {
|
||||
const dir = join(snapshotsDir, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
|
||||
expect(
|
||||
existsSync(join(dir, WINDOWS_STDOUT_SNAPSHOT)),
|
||||
`${name}/${WINDOWS_STDOUT_SNAPSHOT} presence must match \`pinsNativeWindowsStdout\``,
|
||||
).toBe(pinsNativeWindowsStdout === true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
|
||||
.toBe(overridden === true)
|
||||
|
||||
Reference in New Issue
Block a user