feat(dsbench): add SDK evaluation composition
This commit is contained in:
@@ -329,7 +329,7 @@ describe('HarnessSdkServer', () => {
|
||||
agentOptions: { model: 'deepseek' },
|
||||
})
|
||||
const transport = new FakeTransport()
|
||||
const server = new HarnessSdkServer(ctx, transport)
|
||||
const server = new HarnessSdkServer(ctx, transport, { maxTokensAsSuccess: true })
|
||||
|
||||
await settleSubagent(ctx, parentHandle.agent, {
|
||||
provider: 'fork',
|
||||
@@ -355,7 +355,7 @@ describe('HarnessSdkServer', () => {
|
||||
agentId: 'fallback-child-agent',
|
||||
parentSessionId: 'fallback-parent',
|
||||
childSessionId: 'fallback-child-session',
|
||||
status: 'error',
|
||||
status: 'ok',
|
||||
stopReason: 'max-tokens',
|
||||
lastAssistantMessage: [],
|
||||
},
|
||||
@@ -443,6 +443,24 @@ describe('HarnessSdkServer', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('can report max-token turn termination as an accepted evaluation result', async () => {
|
||||
const storageDir = await mkdtemp(join(tmpdir(), 'dsh-jsonrpc-max-tokens-success-'))
|
||||
const ctx = await makeHarness(storageDir)
|
||||
try {
|
||||
const server = new HarnessSdkServer(ctx, new FakeTransport(), { maxTokensAsSuccess: true }) as unknown as {
|
||||
finishedStatus(reason: unknown): 'ok' | 'error'
|
||||
shutdown(): Promise<Record<string, never>>
|
||||
}
|
||||
|
||||
expect(server.finishedStatus({ kind: 'max-tokens' })).toBe('ok')
|
||||
expect(server.finishedStatus({ kind: 'error' })).toBe('error')
|
||||
await server.shutdown()
|
||||
} finally {
|
||||
await ctx.fiber.dispose()
|
||||
await rm(storageDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('reports no adapter when the LLM service is absent', async () => {
|
||||
const ctx = new Context()
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user