Pine/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat: implement bounded LLM request recovery

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-20 03:34:19 +08:00
parent 7cf966fc0e
commit 3b0b0cefeb
115 changed files with 3311 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ui/tui/tests/harness.ts
+3 -2
View File
@@ -120,10 +120,11 @@ export function appendAssistant(
session: Session,
content: ContentBlock[],
usage?: { inputTokens: number; outputTokens: number },
position: { turn: number; step: number } = { turn: 1, step: 0 },
): void {
session.append('assistant/message', {
turn: 1,
step: 0,
turn: position.turn,
step: position.step,
provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
content,
...usage === undefined ? {} : { usage },
packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt
+48
View File
@@ -0,0 +1,48 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=15 bufferRow=15
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Start then cancel. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " Retrying model request (1/2) in 1000ms: temporary transport failure "
style 1-67 fg=yellow
12| <blank>
13| " cancelled during retry delay "
style 1-28 fg=yellow
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
15| " "
style 1-1 inverse
16| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
17| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
18-35| <blank>
packages/ui/tui/tests/snapshots/retry-exhausted.expected.txt
+45
View File
@@ -0,0 +1,45 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=13 bufferRow=13
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Let the bounded policy exhaust. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " provider still unavailable "
style 1-26 fg=red
12| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
13| " "
style 1-1 inverse
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
15| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
16-35| <blank>
packages/ui/tui/tests/snapshots/retry-recovered.expected.txt
+49
View File
@@ -0,0 +1,49 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=16 bufferRow=16
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Recover this request. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " Retrying model request (1/2) in 500ms: provider rate limit "
style 1-58 fg=yellow
12| <blank>
13| " Assistant "
style 1-9 fg=bright-magenta bold
14| " Recovered on the next bounded attempt. "
15| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
16| " "
style 1-1 inverse
17| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
18| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
19-35| <blank>
packages/ui/tui/tests/snapshots/retry-scheduled.expected.txt
+45
View File
@@ -0,0 +1,45 @@
terminal 96x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH snapshot"
cursor hidden column=1 viewportRow=13 bufferRow=13
buffer
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
style 0-95 fg=bright-blue
1| "│ DEEPSEEK HARNESS │"
style 0-0 fg=bright-blue
style 2-9 fg=bright-blue bold
style 11-17 bold
style 95-95 fg=bright-blue
2| "│ Snapshot agent ready. │"
style 0-0 fg=bright-blue
style 2-22 fg=bright-black
style 95-95 fg=bright-blue
3| "│ deepseek-v4-flash • main-session │"
style 0-0 fg=bright-blue
style 2-35 dim
style 95-95 fg=bright-blue
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
style 0-95 fg=bright-blue
5| <blank>
6| "▌ "
style 0-0 fg=bright-blue
7| "▌ You "
style 0-0 fg=bright-blue
style 2-4 fg=bright-blue bold
8| "▌ Recover this request. "
style 0-0 fg=bright-blue
9| "▌ "
style 0-0 fg=bright-blue
10| <blank>
11| " Retrying model request (1/2) in 500ms: provider rate limit "
style 1-58 fg=yellow
12| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
13| " "
style 1-1 inverse
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
style 0-95 dim
15| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
style 0-24 dim
style 63-95 dim
16-35| <blank>
packages/ui/tui/tests/tui.snapshot.ts
+81
View File
@@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'
import { afterAll, describe, expect, it } from 'vitest'
import type { Context } from 'cordis'
import { CallId, type ContentBlock } from '@deepseek-ai/dsh-llm'
import type {} from '@deepseek-ai/dsh-llm-retry'
import type { Session } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry, { type ToolDefinition, type ToolResultView } from '@deepseek-ai/dsh-tools'
@@ -24,6 +25,10 @@ const REFRESHING = process.env.DSH_SNAPSHOT === 'refresh'
const CHECKPOINTS = [
'conversation-streaming',
'retry-scheduled',
'retry-recovered',
'retry-cancelled',
'retry-exhausted',
'code-mode-pending',
'dynamic-workflow-pending',
'cordis-tools-pending',
@@ -222,6 +227,82 @@ describe('TUI terminal-state snapshots', () => {
await disposeSnapshot(harness)
})
it('pins failed-stream retraction, scheduled retry, and eventual success', async () => {
const harness = await setupSnapshot()
await renderAfter(harness, () => {
appendUser(harness.session, 'Recover this request.')
harness.session.append('assistant/chunk', {
turn: 1,
step: 1,
chunk: { type: 'text-delta', index: 0, text: 'discarded partial output' },
})
harness.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 500,
failure: { message: 'provider rate limit', code: 'RATE_LIMIT', status: 429 },
})
})
await checkpoint('retry-scheduled', harness.terminal, { includeScrollback: true })
await renderAfter(harness, () => {
harness.session.append('assistant/message', {
turn: 1,
step: 2,
provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
content: [{ type: 'text', text: 'Recovered on the next bounded attempt.' }],
}, { surfaceOp: 'append' })
harness.session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
})
await checkpoint('retry-recovered', harness.terminal, { includeScrollback: true })
await disposeSnapshot(harness)
})
it('pins cancellation during a scheduled retry delay', async () => {
const harness = await setupSnapshot()
await renderAfter(harness, () => {
appendUser(harness.session, 'Start then cancel.')
harness.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 1_000,
failure: { message: 'temporary transport failure', code: 'TRANSPORT' },
})
harness.session.append('turn/end', {
turn: 1,
reason: { kind: 'aborted', reason: 'cancelled during retry delay' },
})
})
await checkpoint('retry-cancelled', harness.terminal, { includeScrollback: true })
await disposeSnapshot(harness)
})
it('pins terminal exhaustion after retracting a failed partial stream', async () => {
const harness = await setupSnapshot()
await renderAfter(harness, () => {
appendUser(harness.session, 'Let the bounded policy exhaust.')
harness.session.append('assistant/chunk', {
turn: 1,
step: 3,
chunk: { type: 'text-delta', index: 0, text: 'discarded terminal partial output' },
})
harness.session.append('turn/end', {
turn: 1,
reason: {
kind: 'error',
step: 3,
failure: { message: 'provider still unavailable', code: 'SERVER', status: 503 },
},
})
})
await checkpoint('retry-exhausted', harness.terminal, { includeScrollback: true })
await disposeSnapshot(harness)
})
it('pins Code Mode run_code with its production presenter', async () => {
const harness = await setupSnapshot({ configureContext: configureAdvancedTools })
const call = {
packages/ui/tui/tests/tui.spec.ts
+74 -1
View File
@@ -7,6 +7,7 @@ import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import type { ToolDefinition } from '@deepseek-ai/dsh-tools'
import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
import type {} from '@deepseek-ai/dsh-llm-retry'
import {
createTuiChat,
mountTui,
@@ -244,7 +245,12 @@ describe('pi-tui chat lifecycle and transcript', () => {
expect(result.terminal.output).toContain('live thought')
result.terminal.send('\x12')
await tick()
appendAssistant(result.session, [{ type: 'text', text: 'final live answer' }], { inputTokens: 500, outputTokens: 8 })
appendAssistant(
result.session,
[{ type: 'text', text: 'final live answer' }],
{ inputTokens: 500, outputTokens: 8 },
{ turn: 2, step: 0 },
)
await tick()
expect(result.terminal.output).toContain('Working')
@@ -276,6 +282,68 @@ describe('pi-tui chat lifecycle and transcript', () => {
expect(result.terminal.drainInput).toHaveBeenCalledWith(100, 20)
})
it('counts failed and recovered request usage once per step', async () => {
const result = await setup()
result.session.append('assistant/chunk', {
turn: 1,
step: 1,
chunk: { type: 'usage', usage: { inputTokens: 10, outputTokens: 2 } },
})
result.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 500,
failure: { message: 'temporary', code: 'SERVER' },
})
result.session.append('assistant/chunk', {
turn: 1,
step: 2,
chunk: { type: 'usage', usage: { inputTokens: 7, outputTokens: 3 } },
})
appendAssistant(
result.session,
[{ type: 'text', text: 'recovered' }],
{ inputTokens: 7, outputTokens: 3 },
{ turn: 1, step: 2 },
)
await tick()
expect(result.terminal.output).toContain('↑17 ↓5')
await dispose(result)
})
it('retracts a failed live stream and renders its durable retry status', async () => {
const result = await setup()
result.session.append('assistant/chunk', {
turn: 1,
step: 1,
chunk: { type: 'text-delta', index: 0, text: 'discarded partial answer' },
})
result.session.append('llm/retry', {
turn: 1,
step: 1,
retry: 1,
maxRetries: 2,
delayMs: 500,
failure: { message: 'rate limited', code: 'RATE_LIMIT', status: 429 },
})
result.session.append('llm/retry', {
turn: 1,
step: 2,
retry: 2,
maxRetries: 2,
delayMs: 1_000,
failure: { message: 'failed before chunks', code: 'SERVER', status: 503 },
})
await tick()
expect(result.terminal.output).toContain('Retrying model request (1/2) in 500ms: rate limited')
expect(result.terminal.output).toContain('Retrying model request (2/2) in 1000ms: failed before chunks')
await dispose(result)
})
it('renders the ANSI palette and every markdown/content style', async () => {
const result = await setup({
config: { color: true },
@@ -453,10 +521,15 @@ describe('pi-tui chat lifecycle and transcript', () => {
events.session.append('turn/end', { turn: 6, reason: { kind: 'max-tokens' } })
events.session.append('turn/end', { turn: 7, reason: { kind: 'rejected', reason: 'policy' } })
events.session.append('turn/end', { turn: 8, reason: { kind: 'interrupted' } })
events.session.append('turn/end', {
turn: 9,
reason: { kind: 'error', step: 1, failure: { message: 'structured provider failure', code: 'SERVER' } },
})
events.ctx.emit('agent/disposed', events.agent)
await tick()
expect(events.terminal.output).toContain('live failure')
expect(events.terminal.output).toContain('durable failure')
expect(events.terminal.output).toContain('structured provider failure')
expect(events.terminal.output).toContain('stopped')
expect(events.terminal.output).toContain('output-token limit')
expect(events.terminal.output).toContain('Turn rejected')