feat(dsbench): add SDK evaluation composition

This commit is contained in:
Yichen Jiang
2026-07-17 17:29:38 +08:00
parent a6915745e0
commit a38ff125a7
17 changed files with 365 additions and 29 deletions
+20 -2
View File
@@ -329,7 +329,7 @@ describe('HarnessSdkServer', () => {
agentOptions: { model: 'deepseek' },
})
const transport = new FakeTransport()
const server = new HarnessSdkServer(ctx, transport)
const server = new HarnessSdkServer(ctx, transport, { maxTokensAsSuccess: true })
await settleSubagent(ctx, parentHandle.agent, {
provider: 'fork',
@@ -355,7 +355,7 @@ describe('HarnessSdkServer', () => {
agentId: 'fallback-child-agent',
parentSessionId: 'fallback-parent',
childSessionId: 'fallback-child-session',
status: 'error',
status: 'ok',
stopReason: 'max-tokens',
lastAssistantMessage: [],
},
@@ -443,6 +443,24 @@ describe('HarnessSdkServer', () => {
}
})
it('can report max-token turn termination as an accepted evaluation result', async () => {
const storageDir = await mkdtemp(join(tmpdir(), 'dsh-jsonrpc-max-tokens-success-'))
const ctx = await makeHarness(storageDir)
try {
const server = new HarnessSdkServer(ctx, new FakeTransport(), { maxTokensAsSuccess: true }) as unknown as {
finishedStatus(reason: unknown): 'ok' | 'error'
shutdown(): Promise<Record<string, never>>
}
expect(server.finishedStatus({ kind: 'max-tokens' })).toBe('ok')
expect(server.finishedStatus({ kind: 'error' })).toBe('error')
await server.shutdown()
} finally {
await ctx.fiber.dispose()
await rm(storageDir, { recursive: true, force: true })
}
})
it('reports no adapter when the LLM service is absent', async () => {
const ctx = new Context()
try {