Merge remote-tracking branch 'origin/master' into codex/provider-retry-policy
# Conflicts: # .agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md # docs/core-data-structures/llm-streaming.i18n.yaml # packages/llm/llm-retry/README.md # packages/llm/llm-retry/src/index.ts # packages/llm/llm-retry/tests/retry.spec.ts
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import { ProviderRequestId } from '@deepseek-ai/dsh-llm'
|
||||
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
|
||||
import InvariantService from '@deepseek-ai/dsh-invariants'
|
||||
import * as RetryInvariant from '@deepseek-ai/dsh-llm-retry/invariant'
|
||||
@@ -105,6 +106,75 @@ describe('llm-retry invariants', () => {
|
||||
}).toThrow(/always mode must omit maxRetries/)
|
||||
})
|
||||
|
||||
it('validates complete durable failures before either retry mode uses them', async () => {
|
||||
const ctx = await setup()
|
||||
const complete = closeStep(ctx, 'retry-invariant-complete-failure')
|
||||
expect(() => {
|
||||
complete.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
provider: 'mock',
|
||||
mode: 'always',
|
||||
policyKey: alwaysPolicyKey,
|
||||
retry: 1,
|
||||
delayMs: 1,
|
||||
failure: {
|
||||
message: 'provider busy',
|
||||
code: 'RATE_LIMIT',
|
||||
status: 429,
|
||||
providerRetryAfterMs: 25,
|
||||
requestId: ProviderRequestId('request-1'),
|
||||
},
|
||||
})
|
||||
}).not.toThrow()
|
||||
|
||||
const normalNull = closeStep(ctx, 'retry-invariant-normal-null-failure')
|
||||
expect(() => {
|
||||
normalNull.append('llm/retry', {
|
||||
turn: 1, step: 1, ...normal,
|
||||
retry: 1, maxRetries: 2, delayMs: 1, failure: null,
|
||||
} as never)
|
||||
}).toThrow(/failure must be an object/)
|
||||
|
||||
const invalidFailures: readonly [string, unknown, RegExp][] = [
|
||||
['always-null', null, /failure must be an object/],
|
||||
['message-type', { message: 1, code: 'RATE_LIMIT' }, /failure\.message/],
|
||||
['message-empty', { message: '', code: 'RATE_LIMIT' }, /failure\.message/],
|
||||
['code-type', { message: 'failed', code: 1 }, /failure\.code/],
|
||||
['code-empty', { message: 'failed', code: '' }, /failure\.code/],
|
||||
['status-type', { message: 'failed', code: 'RATE_LIMIT', status: 429.5 }, /failure\.status/],
|
||||
['status-low', { message: 'failed', code: 'RATE_LIMIT', status: 99 }, /failure\.status/],
|
||||
['status-high', { message: 'failed', code: 'RATE_LIMIT', status: 600 }, /failure\.status/],
|
||||
[
|
||||
'retry-after-type',
|
||||
{ message: 'failed', code: 'RATE_LIMIT', providerRetryAfterMs: '25' },
|
||||
/failure\.providerRetryAfterMs/,
|
||||
],
|
||||
[
|
||||
'retry-after-zero',
|
||||
{ message: 'failed', code: 'RATE_LIMIT', providerRetryAfterMs: 0 },
|
||||
/failure\.providerRetryAfterMs/,
|
||||
],
|
||||
['request-id-type', { message: 'failed', code: 'RATE_LIMIT', requestId: 1 }, /failure\.requestId/],
|
||||
['request-id-empty', { message: 'failed', code: 'RATE_LIMIT', requestId: '' }, /failure\.requestId/],
|
||||
]
|
||||
for (const [name, invalidFailure, message] of invalidFailures) {
|
||||
const session = closeStep(ctx, `retry-invariant-${name}`)
|
||||
expect(() => {
|
||||
session.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
provider: 'mock',
|
||||
mode: 'always',
|
||||
policyKey: alwaysPolicyKey,
|
||||
retry: 1,
|
||||
delayMs: 1,
|
||||
failure: invalidFailure,
|
||||
} as never)
|
||||
}).toThrow(message)
|
||||
}
|
||||
})
|
||||
|
||||
it('binds event mode and finite budget to the canonical policy key', async () => {
|
||||
const ctx = await setup()
|
||||
const normalModeMismatch = closeStep(ctx, 'retry-invariant-normal-mode-key')
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import type { Fiber } from 'cordis'
|
||||
import LlmService, { CallId, LlmAdapter, LlmError, resolveRetryPolicy } from '@deepseek-ai/dsh-llm'
|
||||
import LlmService, { CallId, EMPTY_RESPONSE_CODE, LlmAdapter, LlmError, resolveRetryPolicy } from '@deepseek-ai/dsh-llm'
|
||||
import type {
|
||||
AlwaysRetryPolicyConfig,
|
||||
BackoffConfig,
|
||||
@@ -74,6 +74,25 @@ function textResponse(text: string): StreamChunk[] {
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* A degenerate empty provider completion as an error finish chunk. Both
|
||||
* adapters emit this shape and the EMPTY_RESPONSE code (the field the policy
|
||||
* routes on); the message text here is the deepseek adapter's phrasing (pi-ai
|
||||
* qualifies it with the model name).
|
||||
*/
|
||||
function emptyCompletion(): StreamChunk[] {
|
||||
return [
|
||||
{ type: 'usage', usage: { inputTokens: 0, outputTokens: 0 } },
|
||||
{
|
||||
type: 'finish',
|
||||
reason: {
|
||||
kind: 'error',
|
||||
failure: { message: 'model returned a completed response with no content', code: EMPTY_RESPONSE_CODE },
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
async function harness(
|
||||
adapter: ScriptedAdapter,
|
||||
policies: Readonly<Record<string, RetryPolicyConfig | undefined>> = { mock: normalConfig() },
|
||||
@@ -184,7 +203,7 @@ describe('provider-routed retry policy', () => {
|
||||
step: 1,
|
||||
provider: 'mock',
|
||||
mode: 'normal',
|
||||
policyKey: '["normal",2,["RATE_LIMIT","SERVER","TIMEOUT","TRANSPORT"],500,10000,0.1]',
|
||||
policyKey: '["normal",2,["EMPTY_RESPONSE","RATE_LIMIT","SERVER","TIMEOUT","TRANSPORT"],500,10000,0.1]',
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 500,
|
||||
@@ -247,6 +266,39 @@ describe('provider-routed retry policy', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('retries an EMPTY_RESPONSE error finish under the default retryable codes', async () => {
|
||||
vi.useFakeTimers()
|
||||
const adapter = new ScriptedAdapter([
|
||||
emptyCompletion(),
|
||||
textResponse('recovered'),
|
||||
])
|
||||
// No retryableCodes override: this proves the default policy covers the
|
||||
// adapters' empty-completion classification end to end (finish-chunk error
|
||||
// delivery, not a thrown stream error).
|
||||
;({ ctx: context } = await harness(adapter))
|
||||
const agent = context.agentLoop.create(SessionId('retry-empty-response'), { provider: 'mock', model: 'mock' })
|
||||
const scheduled = waitForRetry(context, agent, 1)
|
||||
|
||||
agent.followup([{ type: 'text', text: 'go' }])
|
||||
const event = await scheduled
|
||||
expect(event.data.failure).toEqual({
|
||||
message: 'model returned a completed response with no content',
|
||||
code: EMPTY_RESPONSE_CODE,
|
||||
})
|
||||
|
||||
const idle = waitForIdle(context, agent)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
await idle
|
||||
|
||||
expect(adapter.requests).toHaveLength(2)
|
||||
expect(agent.session.events.filter(event => event.type === 'assistant/message').map(event => event.data.step))
|
||||
.toEqual([2])
|
||||
expect(agent.session.deriveMessages().at(-1)).toMatchObject({
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'recovered' }],
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves partial failed chunks on their step without committing a message or tool side effect', async () => {
|
||||
vi.useFakeTimers()
|
||||
const adapter = new ScriptedAdapter([
|
||||
|
||||
Reference in New Issue
Block a user