test(acp): snapshot system prompts as Markdown

This commit is contained in:
Tianyi Cui
2026-07-11 22:24:20 +08:00
parent ee3e4ea8f9
commit 9339622d3b
20 changed files with 543 additions and 135 deletions
@@ -1,2 +1,2 @@
{"type":"session","id":"ccdc749f-56f3-4267-9750-598b5c60b7b2","createdAt":600,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-nOQ4Gy"}
{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
@@ -0,0 +1 @@
SYS PROMPT
@@ -1,3 +1,3 @@
{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"}
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
{"type":"turn/start","seq":1,"time":7,"data":{"turn":1}}
@@ -0,0 +1 @@
SYS PROMPT
@@ -1,5 +1,11 @@
import { describe, expect, it } from 'vitest'
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from '../src/normalize.ts'
import {
type NormalizeContext,
normalizeSessionLog,
normalizeStdout,
scrubRequestHeaders,
scrubSystemPrompts,
} from '../src/normalize.ts'
/**
* Unit tests for the pure snapshot normalizers. Live as a *.spec.ts (runs in
@@ -260,3 +266,43 @@ describe('scrubRequestHeaders', () => {
expect(scrubRequestHeaders(once)).toBe(once)
})
})
describe('scrubSystemPrompts', () => {
it('scrubs only system prompt payloads while keeping tools and prefixes verbatim', () => {
const header = JSON.stringify({
type: 'request/header', seq: 1, time: 2,
data: {
header: {
system: 'full prompt',
tools: [{ name: 'read', description: 'full schema' }],
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'full prefix' }] }],
},
reason: 'initial',
},
})
const delta = JSON.stringify({
type: 'request/header-delta', seq: 2, time: 3,
data: {
system: { keepStart: 1, keepEnd: 2, insert: ['new prompt line'] },
tools: { changed: [{ name: 'read', description: 'changed schema' }] },
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
},
})
const toolsOnly = JSON.stringify({
type: 'request/header', seq: 3, time: 4,
data: { header: { tools: [{ name: 'read', description: 'schema only' }] }, reason: 'resume' },
})
const out = scrubSystemPrompts(`${header}\n${delta}\n${toolsOnly}\n`)
expect(out).toContain('"system":"{{system}}"')
expect(out).toContain('"insert":["{{system}}"]')
expect(out).not.toContain('full prompt')
expect(out).not.toContain('new prompt line')
expect(out).toContain('full schema')
expect(out).toContain('full prefix')
expect(out).toContain('changed schema')
expect(out).toContain('changed prefix')
expect(out.split('\n')[2]).toBe(toolsOnly)
expect(scrubSystemPrompts(out)).toBe(out)
})
})
@@ -8,8 +8,10 @@ import { defineAcpSnapshotSuite, type HarvestedLog, type Scenario } from '../src
import {
childFixturePaths,
fixtureContext,
formatSystemPromptSnapshot,
headerDeltaCount,
normalizedHeaders,
normalizedSystemPrompts,
refreshFixtureReplacements,
stabilizeRefreshLog,
} from '../src/suite.ts'
@@ -78,6 +80,7 @@ afterAll(async () => {
function staleRefreshFixtures(dir: string): void {
writeFileSync(join(dir, 'plain-turn', 'stdout.golden.jsonl'), 'stale stdout\n')
writeFileSync(join(dir, 'pin-turn', 'system-prompt.golden.md'), 'STALE PROMPT\n')
const plainBehaviorFile = join(dir, 'plain-turn', 'behavior.json')
const plainBehavior = JSON.parse(readFileSync(plainBehaviorFile, 'utf8')) as Record<string, unknown>
@@ -124,6 +127,8 @@ describe('defineAcpSnapshotSuite: refresh write-back', () => {
const authored = readFileSync(join(refreshDir, 'authored-error', 'session.jsonl'), 'utf8')
expect(authored).toContain('"error":"model exploded"')
expect(authored).not.toContain('"error":"stale"')
expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.golden.md'), 'utf8')).toBe('SYS PROMPT\n')
})
})
@@ -219,6 +224,28 @@ describe('normalizedHeaders', () => {
})
})
describe('normalizedSystemPrompts', () => {
it('extracts normalized string prompts and omits absent or non-string fields', () => {
const log = [
'{"type":"session","id":"a","createdAt":5,"cwd":"/w"}',
'{"type":"request/header","seq":0,"time":9,"data":{"header":{"system":"work in /w"}}}',
'{"type":"request/header","seq":1,"time":9,"data":{"header":{}}}',
'{"type":"request/header","seq":2,"time":9,"data":{"header":{"system":null}}}',
'{"type":"request/header","seq":3,"time":9,"data":{"header":null}}',
'{"type":"request/header","seq":4,"time":9,"data":{"header":"invalid"}}',
'',
].join('\n')
expect(normalizedSystemPrompts(log, { sessionIds: [], cwd: '/w' })).toEqual(['work in {{cwd}}'])
})
})
describe('formatSystemPromptSnapshot', () => {
it('adds a missing terminal newline without changing an existing one', () => {
expect(formatSystemPromptSnapshot('prompt')).toBe('prompt\n')
expect(formatSystemPromptSnapshot('prompt\n')).toBe('prompt\n')
})
})
describe('headerDeltaCount', () => {
it('counts request/header-delta events, ignoring blanks and other lines', () => {
const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} })