Merge remote-tracking branch 'origin/master' into worktree/pr628-merge-20260727
# Conflicts: # .agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.i18n.yaml # .agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md # .agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.zh.md # docs/architecture.i18n.yaml # docs/architecture.md # docs/architecture.zh.md # docs/cordis-catalog/events.md # docs/core-data-structures/llm-streaming.i18n.yaml # docs/core-data-structures/llm-streaming.md # docs/core-data-structures/llm-streaming.zh.md # docs/event-producer-consumer.md # examples/acp-agent/tests/snapshots/empty-response-retry/session.jsonl # packages/compact/compact-basic/src/index.ts # packages/compact/compact-basic/tests/compact-basic.spec.ts # packages/cordis/tool-cordis/src/api-catalog.ts # packages/core/agent-loop/README.i18n.yaml # packages/core/agent-loop/README.md # packages/core/agent-loop/README.zh.md # packages/core/agent-loop/src/loop.ts # packages/core/agent-loop/tests/request-recovery.spec.ts # packages/core/agent/src/types.ts # packages/core/scope/tests/invariant.spec.ts # packages/llm/llm-retry/README.i18n.yaml # packages/llm/llm-retry/README.md # packages/llm/llm-retry/README.zh.md # packages/llm/llm-retry/src/index.ts # packages/llm/llm-retry/src/invariant.ts # packages/llm/llm-retry/tests/invariant.spec.ts # packages/llm/llm-retry/tests/retry.spec.ts # packages/plan/plan-mode/src/index.ts # packages/plan/plan-mode/tests/integration.spec.ts # packages/plan/plan-mode/tests/plan-mode.spec.ts
This commit is contained in:
@@ -5,8 +5,7 @@ import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
/**
|
||||
* Find the provider in force when one step closed, excluding later recovery mutations.
|
||||
* Request headers remain effective across turn boundaries until a newer full
|
||||
* snapshot changes them. A preceding retry in the same turn is also a route
|
||||
* marker because every provider change requires a newer full snapshot.
|
||||
* snapshot changes them; every provider change requires a newer full snapshot.
|
||||
* @param events - session events containing the closed step.
|
||||
* @param turn - turn that owns the failed step.
|
||||
* @param step - failed step whose provider is required.
|
||||
@@ -28,11 +27,6 @@ export function providerForClosedStep(
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
const event = events[index]!
|
||||
if (event.type === 'request/header') return event.data.header.config.provider
|
||||
if (event.type === 'llm/retry'
|
||||
&& event.data.turn === turn
|
||||
&& event.data.step < step) {
|
||||
return event.data.provider
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
import type { Context } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import type { Agent, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh-agent'
|
||||
import type { Agent, RequestError, RequestErrorAction } from '@deepseek-ai/dsh-agent'
|
||||
import type { LlmFailure, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import { providerForClosedStep } from './history.ts'
|
||||
@@ -63,11 +63,11 @@ export interface RetryInternals {
|
||||
}
|
||||
|
||||
type DownstreamOutcome =
|
||||
| { readonly type: 'decision'; readonly decision: RequestErrorDecision }
|
||||
| { readonly type: 'decision'; readonly decision: RequestErrorAction }
|
||||
| { readonly type: 'error'; readonly error: unknown }
|
||||
|
||||
async function settleDownstream(
|
||||
next: () => Promise<RequestErrorDecision>,
|
||||
next: () => Promise<RequestErrorAction>,
|
||||
): Promise<DownstreamOutcome> {
|
||||
try {
|
||||
return { type: 'decision', decision: await next() }
|
||||
@@ -121,9 +121,9 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
validateConfig(config)
|
||||
const random = internals.random ?? Math.random
|
||||
const lifetime = new AbortController()
|
||||
const active = new Set<Promise<RequestErrorDecision>>()
|
||||
const active = new Set<Promise<RequestErrorAction>>()
|
||||
|
||||
function track(operation: Promise<RequestErrorDecision>): Promise<RequestErrorDecision> {
|
||||
function track(operation: Promise<RequestErrorAction>): Promise<RequestErrorAction> {
|
||||
const tracked = operation.finally(() => active.delete(tracked))
|
||||
active.add(tracked)
|
||||
return tracked
|
||||
@@ -140,9 +140,9 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
retry: number,
|
||||
delayMs: number,
|
||||
signal: AbortSignal,
|
||||
): Promise<RequestErrorDecision> {
|
||||
): Promise<RequestErrorAction> {
|
||||
const fusedSignal = AbortSignal.any([signal, lifetime.signal])
|
||||
if (fusedSignal.aborted) return { action: 'fail' }
|
||||
if (fusedSignal.aborted) return
|
||||
const eventData = policy.mode === 'normal'
|
||||
? {
|
||||
turn,
|
||||
@@ -166,8 +166,8 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
failure,
|
||||
}
|
||||
agent.session.append('llm/retry', eventData)
|
||||
if (!await cancellableDelay(delayMs, fusedSignal)) return { action: 'fail' }
|
||||
return { action: 'retry' }
|
||||
if (!await cancellableDelay(delayMs, fusedSignal)) return
|
||||
return { kind: 'retry' }
|
||||
}
|
||||
|
||||
async function recover(
|
||||
@@ -179,8 +179,8 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
priorFailures: readonly LlmFailure[],
|
||||
policy: ResolvedRetryPolicy | undefined,
|
||||
signal: AbortSignal,
|
||||
next: () => Promise<RequestErrorDecision>,
|
||||
): Promise<RequestErrorDecision> {
|
||||
next: () => Promise<RequestErrorAction>,
|
||||
): Promise<RequestErrorAction> {
|
||||
if (policy === undefined) return next()
|
||||
// The call-local policy belongs to the registration that served this
|
||||
// failure. Recover only the durable provider identity from the header;
|
||||
@@ -191,19 +191,19 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
throw new Error(`llm-retry: no request provider for closed turn ${turn}/step ${step}`)
|
||||
}
|
||||
if (policy.mode === 'always') {
|
||||
if (signal.aborted || lifetime.signal.aborted) return { action: 'fail' }
|
||||
if (signal.aborted || lifetime.signal.aborted) return
|
||||
const fusedSignal = AbortSignal.any([signal, lifetime.signal])
|
||||
// The loop and plugin lifetime stay open until delegated recovery settles.
|
||||
// An abort then wins before the decision or fallback can mutate later state.
|
||||
const downstream = await settleDownstream(next)
|
||||
if (fusedSignal.aborted) return { action: 'fail' }
|
||||
if (fusedSignal.aborted) return
|
||||
if (downstream.type === 'error') {
|
||||
ctx.logger.warn(
|
||||
`llm-retry: provider "${provider}" always policy ignored a downstream recovery failure: %o`,
|
||||
downstream.error,
|
||||
)
|
||||
}
|
||||
if (downstream.type === 'decision' && downstream.decision.action === 'retry') {
|
||||
if (downstream.type === 'decision' && downstream.decision?.kind === 'retry') {
|
||||
return downstream.decision
|
||||
}
|
||||
} else if (!policy.retryableCodes.includes(failure.code)) {
|
||||
@@ -211,12 +211,11 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
}
|
||||
|
||||
const policyKey = retryPolicyKey(policy)
|
||||
const firstPriorStep = step - priorFailures.length
|
||||
const firstPriorTurn = turn - priorFailures.length
|
||||
const priorPolicyRetry = agent.session.events.findLast((event): event is SessionEvent<'llm/retry'> =>
|
||||
event.type === 'llm/retry'
|
||||
&& event.data.turn === turn
|
||||
&& event.data.step >= firstPriorStep
|
||||
&& event.data.step < step
|
||||
&& event.data.turn >= firstPriorTurn
|
||||
&& event.data.turn < turn
|
||||
&& event.data.provider === provider
|
||||
&& event.data.policyKey === policyKey,
|
||||
)
|
||||
@@ -249,12 +248,12 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
|
||||
priorFailures: readonly LlmFailure[],
|
||||
policy: ResolvedRetryPolicy | undefined,
|
||||
signal: AbortSignal,
|
||||
next: () => Promise<RequestErrorDecision>,
|
||||
next: () => Promise<RequestErrorAction>,
|
||||
) => {
|
||||
// A waterfall may have captured this callback before its registration was
|
||||
// removed. Lifetime cancellation must prevent that stale callback from
|
||||
// entering a downstream policy after disposal.
|
||||
if (lifetime.signal.aborted) return Promise.resolve<RequestErrorDecision>({ action: 'fail' })
|
||||
if (lifetime.signal.aborted) return Promise.resolve<RequestErrorAction>(undefined)
|
||||
return track(recover(agent, turn, step, error, failure, priorFailures, policy, signal, next))
|
||||
})
|
||||
|
||||
|
||||
@@ -41,6 +41,34 @@ function validateFailure(value: unknown, fail: InvariantFailure): asserts value
|
||||
}
|
||||
}
|
||||
|
||||
/** Find the first turn in the structured-failure retry chain containing `turn`. */
|
||||
function retryChainStart(history: readonly SessionEvent[], turn: number): number {
|
||||
let startIndex = history.findLastIndex(
|
||||
event => event.type === 'turn/start' && event.data.turn === turn,
|
||||
)
|
||||
while (startIndex >= 0) {
|
||||
const start = history[startIndex]
|
||||
if (start?.type !== 'turn/start' || start.data.trigger.kind !== 'retry') break
|
||||
|
||||
let endIndex = startIndex - 1
|
||||
while (endIndex >= 0 && history[endIndex]?.type !== 'turn/end') endIndex -= 1
|
||||
const end = history[endIndex]
|
||||
if (end?.type !== 'turn/end'
|
||||
|| end.data.reason.kind !== 'error'
|
||||
|| end.data.reason.failure === undefined) break
|
||||
|
||||
const previousStart = history.findLastIndex(
|
||||
(event, index) =>
|
||||
index < endIndex
|
||||
&& event.type === 'turn/start'
|
||||
&& event.data.turn === end.data.turn,
|
||||
)
|
||||
if (previousStart < 0) break
|
||||
startIndex = previousStart
|
||||
}
|
||||
return startIndex
|
||||
}
|
||||
|
||||
/** Validate one retry record against the open turn and most recently closed step. */
|
||||
function validateRetry(
|
||||
history: readonly SessionEvent[],
|
||||
@@ -78,20 +106,23 @@ function validateRetry(
|
||||
fail(`llm/retry delayMs must be a finite number within 0..${MAX_TIMER_DELAY_MS}`)
|
||||
}
|
||||
|
||||
const turnStartIndex = history.findLastIndex(prior =>
|
||||
prior.type === 'turn/start' || prior.type === 'turn/end')
|
||||
const turnBoundary = history[turnStartIndex]
|
||||
if (turnBoundary?.type !== 'turn/start') {
|
||||
fail('llm/retry must be appended inside an open turn')
|
||||
const currentTurnEvents: SessionEvent[] = []
|
||||
let openTurn: number | undefined
|
||||
for (const prior of history.slice().reverse()) {
|
||||
if (prior.type === 'turn/end') fail('llm/retry must be appended inside an open turn')
|
||||
if (prior.type === 'turn/start') {
|
||||
openTurn = prior.data.turn
|
||||
break
|
||||
}
|
||||
currentTurnEvents.push(prior)
|
||||
}
|
||||
const openTurn = turnBoundary.data.turn
|
||||
if (openTurn === undefined) fail('llm/retry must be appended inside an open turn')
|
||||
if (turn !== openTurn) {
|
||||
fail(`llm/retry names turn ${turn}, but the open turn is ${openTurn}`)
|
||||
}
|
||||
|
||||
const currentTurnEvents = history.slice(turnStartIndex + 1)
|
||||
let closedStep: number | undefined
|
||||
for (const prior of currentTurnEvents.slice().reverse()) {
|
||||
for (const prior of currentTurnEvents) {
|
||||
if (prior.type === 'step/start') {
|
||||
fail(`llm/retry must follow step/end, but step ${prior.data.step} is still open`)
|
||||
}
|
||||
@@ -108,18 +139,16 @@ function validateRetry(
|
||||
fail(`llm/retry provider ${provider} does not match the failed request provider ${String(routedProvider)}`)
|
||||
}
|
||||
|
||||
const priorRetries = currentTurnEvents
|
||||
const chainStart = retryChainStart(history, turn)
|
||||
const chain = history.slice(Math.max(chainStart, 0))
|
||||
const lastSuccess = chain.findLastIndex(prior => prior.type === 'assistant/message')
|
||||
const chainRetries = chain.slice(lastSuccess + 1)
|
||||
.filter((prior): prior is SessionEvent<'llm/retry'> => prior.type === 'llm/retry')
|
||||
if (priorRetries.some(prior => prior.data.step === step)) {
|
||||
if (chainRetries.some(prior => prior.data.turn === turn && prior.data.step === step)) {
|
||||
fail(`llm/retry duplicates the retry record for turn ${turn}/step ${step}`)
|
||||
}
|
||||
const lastSuccessIndex = currentTurnEvents.findLastIndex(prior => prior.type === 'assistant/message')
|
||||
const priorPolicyRetry = currentTurnEvents.findLast((prior, index): prior is SessionEvent<'llm/retry'> => (
|
||||
index > lastSuccessIndex
|
||||
&& prior.type === 'llm/retry'
|
||||
&& prior.data.provider === provider
|
||||
&& prior.data.policyKey === policyKey
|
||||
))
|
||||
const priorPolicyRetry = chainRetries.findLast(prior =>
|
||||
prior.data.provider === provider && prior.data.policyKey === policyKey)
|
||||
const expectedRetry = (priorPolicyRetry?.data.retry ?? 0) + 1
|
||||
if (retry !== expectedRetry) {
|
||||
fail(`llm/retry retry ${retry} must equal provider policy retry ${expectedRetry}`)
|
||||
|
||||
Reference in New Issue
Block a user