Pine/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat: implement bounded LLM request recovery

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-20 03:34:19 +08:00
parent 7cf966fc0e
commit 3b0b0cefeb
115 changed files with 3311 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ui/tui/README.md
+1 -1
View File
@@ -6,7 +6,7 @@ The implemented [TUI feature Agent Note](../../../.agents/notes/implemented/feat
This package owns interactive terminal presentation and input only. It injects `agents`, `tools`, and `userInteraction`, then drives an agent created or resumed by app or developer code. Agent lifecycle, persistence, and the model-facing [`ask_user_question`](../tool-ask-user/README.md) tool remain separate composition entries.
The TUI rebuilds resumed history from the active session surface, renders Markdown responses and reasoning, applies each tool's `presentCall` / `presentResult` intent to terminal, diff, or generic cards, keeps the latest `todo/write` plan above the editor, and presents `ctx.userInteraction` questions as keyboard-driven overlays. Surface replacement events rebuild the transcript so compacted history does not reappear.
The TUI rebuilds resumed history from the active session surface, renders Markdown responses and reasoning, applies each tool's `presentCall` / `presentResult` intent to terminal, diff, or generic cards, keeps the latest `todo/write` plan above the editor, and presents `ctx.userInteraction` questions as keyboard-driven overlays. A durable `llm/retry` event retracts the failed step's live chunks and renders the scheduled retry count, delay, and failure in the transcript; success, exhaustion, and cancellation then settle through ordinary session events. The footer totals each logged model step's usage once, including failed attempts, while treating committed-message usage as a fallback for logs without a usage chunk. Surface replacement events rebuild the transcript so compacted history does not reappear.
Before model output, session events, tool presenters, questions, configuration, or diagnostics reach pi-tui's ANSI-aware renderers or the terminal title, the TUI renders C0 and C1 controls other than line feeds as visible `\xNN` text. Those sources cannot add terminal control sequences; the TUI and pi-tui retain ownership of terminal rendering and styling.
packages/ui/tui/package.json
+2
View File
@@ -25,6 +25,7 @@
"@deepseek-ai/dsh-agent": "^0.0.1",
"@deepseek-ai/dsh-agent-loop": "^0.0.1",
"@deepseek-ai/dsh-llm": "^0.0.1",
"@deepseek-ai/dsh-llm-retry": "^0.0.1",
"@deepseek-ai/dsh-session": "^0.0.1",
"@deepseek-ai/dsh-tools": "^0.0.1",
"@deepseek-ai/dsh-user-interaction": "^0.0.1",
@@ -39,6 +40,7 @@
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-agent-loop": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-llm-retry": "workspace:^",
"@deepseek-ai/dsh-session": "workspace:^",
"@deepseek-ai/dsh-system-prompt": "workspace:^",
"@deepseek-ai/dsh-tool-cordis": "workspace:^",
packages/ui/tui/src/index.ts
+56 -19
View File
@@ -35,7 +35,8 @@ import type { Context } from 'cordis'
import z from 'schemastery'
import type { Agent, AgentStatus } from '@deepseek-ai/dsh-agent'
import type {} from '@deepseek-ai/dsh-agent-loop'
import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, StreamChunk, TokenUsage } from '@deepseek-ai/dsh-llm'
import type {} from '@deepseek-ai/dsh-llm-retry'
import { SessionId, type Session, type SessionEvent, type TodoItem } from '@deepseek-ai/dsh-session'
import type {
FileDiff,
@@ -612,15 +613,38 @@ function formatCwd(cwd: string | undefined): string {
return displayText(cwd)
}
function sessionTokens(session: Session): { input: number; output: number } {
let input = 0
let output = 0
for (const event of session.events) {
if (event.type !== 'assistant/message' || event.data.usage === undefined) continue
input += event.data.usage.inputTokens
output += event.data.usage.outputTokens
interface SessionTokenTotals {
input: number
output: number
readonly byStep: Map<string, TokenUsage>
}
function recordTokenUsage(totals: SessionTokenTotals, turn: number, step: number, usage: TokenUsage): void {
const key = `${turn}:${step}`
const previous = totals.byStep.get(key)
if (previous !== undefined) {
totals.input -= previous.inputTokens
totals.output -= previous.outputTokens
}
return { input, output }
totals.byStep.set(key, usage)
totals.input += usage.inputTokens
totals.output += usage.outputTokens
}
function recordEventUsage(totals: SessionTokenTotals, event: SessionEvent): void {
if (event.type === 'assistant/chunk' && event.data.chunk.type === 'usage') {
recordTokenUsage(totals, event.data.turn, event.data.step, event.data.chunk.usage)
} else if (event.type === 'assistant/message' && event.data.usage !== undefined) {
recordTokenUsage(totals, event.data.turn, event.data.step, event.data.usage)
}
}
function sessionTokens(session: Session): SessionTokenTotals {
const totals: SessionTokenTotals = { input: 0, output: 0, byStep: new Map() }
for (const event of session.events) {
recordEventUsage(totals, event)
}
return totals
}
class FooterComponent implements Component {
@@ -899,6 +923,14 @@ export function createTuiChat(
return card
}
const clearStreaming = (): void => {
if (streaming === undefined) return
const index = chat.children.indexOf(streaming)
/* v8 ignore next -- streaming is assigned only after the same component is added, and every removal clears it. */
if (index >= 0) chat.children.splice(index, 1)
streaming = undefined
}
const renderEvent = (event: SessionEvent, options: { addHistory: boolean; renderChunks: boolean }): void => {
switch (event.type) {
case 'user/message': {
@@ -941,15 +973,19 @@ export function createTuiChat(
}
break
case 'assistant/message': {
if (streaming !== undefined) {
const index = chat.children.indexOf(streaming)
if (index >= 0) chat.children.splice(index, 1)
streaming = undefined
}
clearStreaming()
const component = new AssistantMessageComponent(event.data.content, showReasoning, palette, mdTheme)
if (component.children.length > 0) chat.addChild(component)
break
}
case 'llm/retry': {
clearStreaming()
appendNotice(
`Retrying model request (${event.data.retry}/${event.data.maxRetries}) in ${event.data.delayMs}ms: ${event.data.failure.message}`,
'warning',
)
break
}
case 'tool/call':
chat.addChild(new Spacer(1))
chat.addChild(parsedTool(event))
@@ -970,9 +1006,13 @@ export function createTuiChat(
todo.update(event.data.todos)
break
case 'turn/end':
clearStreaming()
if (event.data.reason.kind === 'error') {
const key = `${event.data.turn}:${event.data.reason.step}`
if (!liveErrors.delete(key)) appendNotice(event.data.reason.message, 'error')
const message = 'failure' in event.data.reason
? event.data.reason.failure.message
: event.data.reason.message
if (!liveErrors.delete(key)) appendNotice(message, 'error')
} else if (event.data.reason.kind === 'aborted') {
appendNotice(event.data.reason.reason ?? 'Turn cancelled.', 'warning')
} else if (event.data.reason.kind === 'max-tokens') {
@@ -1245,10 +1285,7 @@ export function createTuiChat(
const disposeSessionEvents = ctx.on('session/event', (session, event) => {
if (session !== agent.session) return
if (event.type === 'assistant/message' && event.data.usage !== undefined) {
tokens.input += event.data.usage.inputTokens
tokens.output += event.data.usage.outputTokens
}
recordEventUsage(tokens, event)
if ('surfaceOp' in event && typeof event.surfaceOp === 'object') {
rebuildTranscript(false)
return
packages/ui/tui/tests/harness.ts
+3 -2
View File
@@ -120,10 +120,11 @@ export function appendAssistant(
session: Session,
content: ContentBlock[],
usage?: { inputTokens: number; outputTokens: number },
position: { turn: number; step: number } = { turn: 1, step: 0 },
): void {
session.append('assistant/message', {
turn: 1,
step: 0,
turn: position.turn,
step: position.step,
provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
content,
...usage === undefined ? {} : { usage },
packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt
+48
View File
@@ -0,0 +1,48 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=15 bufferRow=15
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Start then cancel. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " Retrying model request (1/2) in 1000ms: temporary transport failure "
style 1-67 fg=yellow
12| <blank>
13| " cancelled during retry delay "
style 1-28 fg=yellow
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
15| " "
style 1-1 inverse
16| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
17| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
18-35| <blank>
packages/ui/tui/tests/snapshots/retry-exhausted.expected.txt
+45
View File
@@ -0,0 +1,45 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=13 bufferRow=13
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Let the bounded policy exhaust. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " provider still unavailable "
style 1-26 fg=red
12| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
13| " "
style 1-1 inverse
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
15| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
16-35| <blank>
packages/ui/tui/tests/snapshots/retry-recovered.expected.txt
+49
View File
@@ -0,0 +1,49 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=16 bufferRow=16
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Recover this request. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " Retrying model request (1/2) in 500ms: provider rate limit "
style 1-58 fg=yellow
12| <blank>
13| " Assistant "
style 1-9 fg=bright-magenta bold
14| " Recovered on the next bounded attempt. "
15| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
16| " "
style 1-1 inverse
17| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
18| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
19-35| <blank>
packages/ui/tui/tests/snapshots/retry-scheduled.expected.txt
+45
View File
@@ -0,0 +1,45 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=13 bufferRow=13
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Recover this request. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " Retrying model request (1/2) in 500ms: provider rate limit "
style 1-58 fg=yellow
12| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
13| " "
style 1-1 inverse
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
15| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
16-35| <blank>
packages/ui/tui/tests/tui.snapshot.ts
+81
View File
@@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'
import { afterAll, describe, expect, it } from 'vitest'
import type { Context } from 'cordis'
import { CallId, type ContentBlock } from '@deepseek-ai/dsh-llm'
import type {} from '@deepseek-ai/dsh-llm-retry'
import type { Session } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry, { type ToolDefinition, type ToolResultView } from '@deepseek-ai/dsh-tools'
@@ -24,6 +25,10 @@ const REFRESHING = process.env.DSH_SNAPSHOT === 'refresh'
const CHECKPOINTS = [
'conversation-streaming',
'retry-scheduled',
'retry-recovered',
'retry-cancelled',
'retry-exhausted',
'code-mode-pending',
'dynamic-workflow-pending',
'cordis-tools-pending',
@@ -222,6 +227,82 @@ describe('TUI terminal-state snapshots', () => {
await disposeSnapshot(harness)
})
it('pins failed-stream retraction, scheduled retry, and eventual success', async () => {
const harness = await setupSnapshot()
await renderAfter(harness, () => {
appendUser(harness.session, 'Recover this request.')
harness.session.append('assistant/chunk', {
turn: 1,
step: 1,
chunk: { type: 'text-delta', index: 0, text: 'discarded partial output' },
})
harness.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 500,
failure: { message: 'provider rate limit', code: 'RATE_LIMIT', status: 429 },
})
})
await checkpoint('retry-scheduled', harness.terminal, { includeScrollback: true })
await renderAfter(harness, () => {
harness.session.append('assistant/message', {
turn: 1,
step: 2,
provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
content: [{ type: 'text', text: 'Recovered on the next bounded attempt.' }],
}, { surfaceOp: 'append' })
harness.session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
})
await checkpoint('retry-recovered', harness.terminal, { includeScrollback: true })
await disposeSnapshot(harness)
})
it('pins cancellation during a scheduled retry delay', async () => {
const harness = await setupSnapshot()
await renderAfter(harness, () => {
appendUser(harness.session, 'Start then cancel.')
harness.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 1_000,
failure: { message: 'temporary transport failure', code: 'TRANSPORT' },
})
harness.session.append('turn/end', {
turn: 1,
reason: { kind: 'aborted', reason: 'cancelled during retry delay' },
})
})
await checkpoint('retry-cancelled', harness.terminal, { includeScrollback: true })
await disposeSnapshot(harness)
})
it('pins terminal exhaustion after retracting a failed partial stream', async () => {
const harness = await setupSnapshot()
await renderAfter(harness, () => {
appendUser(harness.session, 'Let the bounded policy exhaust.')
harness.session.append('assistant/chunk', {
turn: 1,
step: 3,
chunk: { type: 'text-delta', index: 0, text: 'discarded terminal partial output' },
})
harness.session.append('turn/end', {
turn: 1,
reason: {
kind: 'error',
step: 3,
failure: { message: 'provider still unavailable', code: 'SERVER', status: 503 },
},
})
})
await checkpoint('retry-exhausted', harness.terminal, { includeScrollback: true })
await disposeSnapshot(harness)
})
it('pins Code Mode run_code with its production presenter', async () => {
const harness = await setupSnapshot({ configureContext: configureAdvancedTools })
const call = {
packages/ui/tui/tests/tui.spec.ts
+74 -1
View File
@@ -7,6 +7,7 @@ import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import type { ToolDefinition } from '@deepseek-ai/dsh-tools'
import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
import type {} from '@deepseek-ai/dsh-llm-retry'
import {
createTuiChat,
mountTui,
@@ -244,7 +245,12 @@ describe('pi-tui chat lifecycle and transcript', () => {
expect(result.terminal.output).toContain('live thought')
result.terminal.send('\x12')
await tick()
appendAssistant(result.session, [{ type: 'text', text: 'final live answer' }], { inputTokens: 500, outputTokens: 8 })
appendAssistant(
result.session,
[{ type: 'text', text: 'final live answer' }],
{ inputTokens: 500, outputTokens: 8 },
{ turn: 2, step: 0 },
)
await tick()
expect(result.terminal.output).toContain('Working')
@@ -276,6 +282,68 @@ describe('pi-tui chat lifecycle and transcript', () => {
expect(result.terminal.drainInput).toHaveBeenCalledWith(100, 20)
})
it('counts failed and recovered request usage once per step', async () => {
const result = await setup()
result.session.append('assistant/chunk', {
turn: 1,
step: 1,
chunk: { type: 'usage', usage: { inputTokens: 10, outputTokens: 2 } },
})
result.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 500,
failure: { message: 'temporary', code: 'SERVER' },
})
result.session.append('assistant/chunk', {
turn: 1,
step: 2,
chunk: { type: 'usage', usage: { inputTokens: 7, outputTokens: 3 } },
})
appendAssistant(
result.session,
[{ type: 'text', text: 'recovered' }],
{ inputTokens: 7, outputTokens: 3 },
{ turn: 1, step: 2 },
)
await tick()
expect(result.terminal.output).toContain('↑17 ↓5')
await dispose(result)
})
it('retracts a failed live stream and renders its durable retry status', async () => {
const result = await setup()
result.session.append('assistant/chunk', {
turn: 1,
step: 1,
chunk: { type: 'text-delta', index: 0, text: 'discarded partial answer' },
})
result.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 500,
failure: { message: 'rate limited', code: 'RATE_LIMIT', status: 429 },
})
result.session.append('llm/retry', {
turn: 1,
step: 2,
retry: 2,
maxRetries: 2,
delayMs: 1_000,
failure: { message: 'failed before chunks', code: 'SERVER', status: 503 },
})
await tick()
expect(result.terminal.output).toContain('Retrying model request (1/2) in 500ms: rate limited')
expect(result.terminal.output).toContain('Retrying model request (2/2) in 1000ms: failed before chunks')
await dispose(result)
})
it('renders the ANSI palette and every markdown/content style', async () => {
const result = await setup({
config: { color: true },
@@ -453,10 +521,15 @@ describe('pi-tui chat lifecycle and transcript', () => {
events.session.append('turn/end', { turn: 6, reason: { kind: 'max-tokens' } })
events.session.append('turn/end', { turn: 7, reason: { kind: 'rejected', reason: 'policy' } })
events.session.append('turn/end', { turn: 8, reason: { kind: 'interrupted' } })
events.session.append('turn/end', {
turn: 9,
reason: { kind: 'error', step: 1, failure: { message: 'structured provider failure', code: 'SERVER' } },
})
events.ctx.emit('agent/disposed', events.agent)
await tick()
expect(events.terminal.output).toContain('live failure')
expect(events.terminal.output).toContain('durable failure')
expect(events.terminal.output).toContain('structured provider failure')
expect(events.terminal.output).toContain('stopped')
expect(events.terminal.output).toContain('output-token limit')
expect(events.terminal.output).toContain('Turn rejected')
packages/ui/tui/tsconfig.json
+3
View File
@@ -26,6 +26,9 @@
{
"path": "../../llm/llm"
},
{
"path": "../../llm/llm-retry"
},
{
"path": "../../core/tools"
},