Pine/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

test(hooks): snapshot the CC + Codex hook matrix end-to-end

Browse Source 
Load both hook bridges in the ACP example (dsh-hooks-claude → ./hooks.json,
dsh-hooks-codex → ./codex-hooks.json) so the full-transcript snapshot tier can
exercise each dialect against the real app. An absent config file is a silent
no-op, so a scenario carries only the file it needs and the other bridge
vanishes — verified byte-identical against every pre-existing snapshot.

Add a scenario per hook point × its headline Decision outcome, both dialects:
UserPromptSubmit block (authored, keyless) + context-fold, PreToolUse deny/ask,
PostToolUse block/context, Stop force-continue. The mid-turn scenarios are
recorded against the real API with the hook active, so the model's reaction to
a denied/blocked/force-continued turn is part of the replayed transcript.

SessionStart and SubagentStart are deliberately excluded (detached best-effort
inject races the log position — a recorded golden fails 10/10 on its own
replay), as is SubagentStop (observe-only, zero transcript footprint — a golden
could never be proven to fail). Both stay on the bridges' unit coverage. See
docs/rfc/implemented/testing/2026-07-04-hook-snapshot-matrix.md.
This commit is contained in:
Tianyi Cui
2026-07-04 00:40:35 +08:00
parent 907c8d3871
commit 74f61c89e3
58 changed files with 3860 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/acp-agent/tests/snapshots/hook-cc-promptsubmit-context/input.json
+7
View File
@@ -0,0 +1,7 @@
{
"steps": [
{ "op": "initialize" },
{ "op": "newSession" },
{ "op": "prompt", "text": "What is my favorite color? Reply with just the color and stop. Do not use any tools." }
]
}
examples/acp-agent/tests/snapshots/hook-cc-promptsubmit-context/session.jsonl
+60
View File
@@ -0,0 +1,60 @@
{"type":"session","version":0,"id":"b0c9d2c7-f95b-4750-be8a-10121253b006","createdAt":1783095036603,"cwd":"/tmp/acp-snap-cwd-LW2rSZ"}
{"type":"turn/start","seq":0,"time":1783095036609,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}
{"type":"hook/invoked","seq":1,"time":1783095036610,"data":{"turn":1,"point":"UserPromptSubmit","dialect":"claude","handlerId":"claude:UserPromptSubmit:1"}}
{"type":"hook/result","seq":2,"time":1783095036623,"data":{"turn":1,"point":"UserPromptSubmit","handlerId":"claude:UserPromptSubmit:1","decision":"pass","exitCode":0,"durationMs":11.874454999999898}}
{"type":"user/message","seq":3,"time":1783095036623,"data":{"content":[{"type":"text","text":"What is my favorite color? Reply with just the color and stop. Do not use any tools."}],"source":{"kind":"user"}},"surfaceOp":"append"}
{"type":"context/message","seq":4,"time":1783095036623,"data":{"content":[{"type":"text","text":"The user has previously stated their favorite color is teal."}],"source":{"kind":"plugin","plugin":"hooks-claude"}},"surfaceOp":"append"}
{"type":"step/start","seq":5,"time":1783095036624,"data":{"turn":1,"step":1}}
{"type":"assistant/chunk","seq":6,"time":1783095037385,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
{"type":"assistant/chunk","seq":7,"time":1783095037385,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}}
{"type":"assistant/chunk","seq":8,"time":1783095037558,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}}
{"type":"assistant/chunk","seq":9,"time":1783095037617,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" is"}}}
{"type":"assistant/chunk","seq":10,"time":1783095037645,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" asking"}}}
{"type":"assistant/chunk","seq":11,"time":1783095037646,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" about"}}}
{"type":"assistant/chunk","seq":12,"time":1783095037669,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" their"}}}
{"type":"assistant/chunk","seq":13,"time":1783095037669,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" favorite"}}}
{"type":"assistant/chunk","seq":14,"time":1783095037669,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" color"}}}
{"type":"assistant/chunk","seq":15,"time":1783095037669,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}}
{"type":"assistant/chunk","seq":16,"time":1783095037691,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" and"}}}
{"type":"assistant/chunk","seq":17,"time":1783095037692,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}}
{"type":"assistant/chunk","seq":18,"time":1783095037692,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" context"}}}
{"type":"assistant/chunk","seq":19,"time":1783095037692,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" from"}}}
{"type":"assistant/chunk","seq":20,"time":1783095037692,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" a"}}}
{"type":"assistant/chunk","seq":21,"time":1783095037714,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" plugin"}}}
{"type":"assistant/chunk","seq":22,"time":1783095037714,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" indicates"}}}
{"type":"assistant/chunk","seq":23,"time":1783095037736,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" they"}}}
{"type":"assistant/chunk","seq":24,"time":1783095037737,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" previously"}}}
{"type":"assistant/chunk","seq":25,"time":1783095037760,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" stated"}}}
{"type":"assistant/chunk","seq":26,"time":1783095037760,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}}
{"type":"assistant/chunk","seq":27,"time":1783095037783,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"'s"}}}
{"type":"assistant/chunk","seq":28,"time":1783095037783,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" te"}}}
{"type":"assistant/chunk","seq":29,"time":1783095037783,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"al"}}}
{"type":"assistant/chunk","seq":30,"time":1783095037783,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}}
{"type":"assistant/chunk","seq":31,"time":1783095037783,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" They"}}}
{"type":"assistant/chunk","seq":32,"time":1783095037807,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" explicitly"}}}
{"type":"assistant/chunk","seq":33,"time":1783095037830,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" asked"}}}
{"type":"assistant/chunk","seq":34,"time":1783095037854,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}}
{"type":"assistant/chunk","seq":35,"time":1783095037854,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}}
{"type":"assistant/chunk","seq":36,"time":1783095037876,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}}
{"type":"assistant/chunk","seq":37,"time":1783095037876,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}}
{"type":"assistant/chunk","seq":38,"time":1783095037899,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}}
{"type":"assistant/chunk","seq":39,"time":1783095037899,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}}
{"type":"assistant/chunk","seq":40,"time":1783095037899,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" color"}}}
{"type":"assistant/chunk","seq":41,"time":1783095037899,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" and"}}}
{"type":"assistant/chunk","seq":42,"time":1783095037899,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" stop"}}}
{"type":"assistant/chunk","seq":43,"time":1783095037900,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}}
{"type":"assistant/chunk","seq":44,"time":1783095037923,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" without"}}}
{"type":"assistant/chunk","seq":45,"time":1783095037923,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" using"}}}
{"type":"assistant/chunk","seq":46,"time":1783095037923,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" any"}}}
{"type":"assistant/chunk","seq":47,"time":1783095037923,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tools"}}}
{"type":"assistant/chunk","seq":48,"time":1783095037923,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}}
{"type":"assistant/chunk","seq":49,"time":1783095037945,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}}
{"type":"assistant/chunk","seq":50,"time":1783095037945,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"te"}}}
{"type":"assistant/chunk","seq":51,"time":1783095037946,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"al"}}}
{"type":"assistant/chunk","seq":52,"time":1783095037946,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user is asking about their favorite color, and the context from a plugin indicates they previously stated it's teal. They explicitly asked me to reply with just the color and stop, without using any tools."}}}}
{"type":"assistant/chunk","seq":53,"time":1783095037946,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"teal"}}}}
{"type":"assistant/chunk","seq":54,"time":1783095037946,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":86,"outputTokens":45,"cacheReadTokens":1664,"reasoningTokens":42}}}}
{"type":"assistant/chunk","seq":55,"time":1783095037946,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
{"type":"assistant/message","seq":56,"time":1783095037948,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user is asking about their favorite color, and the context from a plugin indicates they previously stated it's teal. They explicitly asked me to reply with just the color and stop, without using any tools."},{"type":"text","text":"teal"}],"usage":{"inputTokens":86,"outputTokens":45,"cacheReadTokens":1664,"reasoningTokens":42}},"sourceEventSeqs":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55],"surfaceOp":"append"}
{"type":"step/end","seq":57,"time":1783095037948,"data":{"turn":1,"step":1}}
{"type":"turn/end","seq":58,"time":1783095037948,"data":{"turn":1,"reason":{"kind":"completed"}}}
examples/acp-agent/tests/snapshots/hook-cc-promptsubmit-context/stdout.golden.jsonl
+47
View File
@@ -0,0 +1,47 @@
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}}
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" user"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" is"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" asking"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" about"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" their"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" favorite"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" color"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":","}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" and"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" the"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" context"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" from"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" a"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" plugin"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" indicates"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" they"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" previously"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" stated"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" it"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"'s"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" te"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"al"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" They"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" explicitly"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" asked"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" me"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" to"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" reply"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" with"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" just"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" the"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" color"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" and"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" stop"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":","}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" without"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" using"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" any"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" tools"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"te"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"al"}}}}
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
examples/acp-agent/tests/snapshots/hook-cc-promptsubmit-context/workspace/hooks.json
+11
View File
@@ -0,0 +1,11 @@
{
"hooks": {
"UserPromptSubmit": [
{
"hooks": [
{ "type": "command", "command": "echo '{\"hookSpecificOutput\":{\"hookEventName\":\"UserPromptSubmit\",\"additionalContext\":\"The user has previously stated their favorite color is teal.\"}}'" }
]
}
]
}
}