feat(llm): add scriptable mock fault server
This commit is contained in:
Yichen Jiang
@@ -0,0 +1,121 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
MOCK_LLM_CLI_USAGE,
|
||||
parseMockLlmCliArgs,
|
||||
} from '../src/cli.ts'
|
||||
|
||||
describe('mock LLM server CLI parser', () => {
|
||||
it('returns help without requiring a sequence', () => {
|
||||
expect(parseMockLlmCliArgs(['--help'])).toEqual({ kind: 'help' })
|
||||
expect(MOCK_LLM_CLI_USAGE).toContain('--sequence')
|
||||
})
|
||||
|
||||
it('parses every request and listener option', () => {
|
||||
expect(parseMockLlmCliArgs([
|
||||
'--sequence', 'connection_refused,partial_disconnect,success',
|
||||
'--host', 'localhost',
|
||||
'--port', '9010',
|
||||
'--api-key', 'mock-key',
|
||||
'--listen-delay-ms', '100',
|
||||
'--repeat-last',
|
||||
'--success-text', 'done',
|
||||
'--partial-text', 'half',
|
||||
'--reasoning-text', 'think',
|
||||
'--chunk-size', '2',
|
||||
'--chunk-delay-ms', '3',
|
||||
'--disconnect-delay-ms', '4',
|
||||
'--retry-after-ms', '5000',
|
||||
'--request-id', 'request-1',
|
||||
'--tool-name', 'lookup',
|
||||
'--tool-arguments', '{"id":1}',
|
||||
])).toEqual({
|
||||
kind: 'run',
|
||||
config: {
|
||||
startsUnavailable: true,
|
||||
listenDelayMs: 100,
|
||||
server: {
|
||||
sequence: ['partial_disconnect', 'success'],
|
||||
host: 'localhost',
|
||||
port: 9010,
|
||||
apiKey: 'mock-key',
|
||||
repeatLast: true,
|
||||
successText: 'done',
|
||||
partialText: 'half',
|
||||
reasoningText: 'think',
|
||||
chunkSize: 2,
|
||||
chunkDelayMs: 3,
|
||||
disconnectDelayMs: 4,
|
||||
retryAfterMs: 5000,
|
||||
requestId: 'request-1',
|
||||
toolName: 'lookup',
|
||||
toolArguments: '{"id":1}',
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('uses standalone defaults for an ordinary sequence', () => {
|
||||
expect(parseMockLlmCliArgs(['--sequence', 'success'])).toEqual({
|
||||
kind: 'run',
|
||||
config: {
|
||||
startsUnavailable: false,
|
||||
listenDelayMs: 0,
|
||||
server: {
|
||||
sequence: ['success'],
|
||||
port: 8000,
|
||||
repeatLast: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('uses the default unavailable interval', () => {
|
||||
const result = parseMockLlmCliArgs(['--sequence', 'connection_refused,success', '--port', '8001'])
|
||||
expect(result).toMatchObject({
|
||||
kind: 'run',
|
||||
config: { startsUnavailable: true, listenDelayMs: 750 },
|
||||
})
|
||||
})
|
||||
|
||||
it('parses a reproducible weighted random profile', () => {
|
||||
expect(parseMockLlmCliArgs([
|
||||
'--sequence', 'random',
|
||||
'--repeat-last',
|
||||
'--seed', '42',
|
||||
'--random-weights', 'success=8,partial_disconnect=2',
|
||||
])).toEqual({
|
||||
kind: 'run',
|
||||
config: {
|
||||
startsUnavailable: false,
|
||||
listenDelayMs: 0,
|
||||
server: {
|
||||
sequence: ['random'],
|
||||
port: 8000,
|
||||
repeatLast: true,
|
||||
randomSeed: 42,
|
||||
randomWeights: { success: 8, partial_disconnect: 2 },
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
[[], /--sequence is required/],
|
||||
[['--wat'], /requires a value/],
|
||||
[['--wat', 'x'], /unknown option/],
|
||||
[['--port', 'NaN', '--sequence', 'success'], /finite number/],
|
||||
[['--sequence', 'success,'], /non-empty/],
|
||||
[['--sequence', 'success,connection_refused'], /only as the first/],
|
||||
[['--sequence', 'connection_refused'], /must be followed/],
|
||||
[['--sequence', 'unknown'], /unknown behavior/],
|
||||
[['--sequence', 'connection_refused,success', '--port', '0'], /nonzero/],
|
||||
[['--sequence', 'success', '--listen-delay-ms', '5'], /requires connection_refused/],
|
||||
[['--sequence', 'success', '--seed', '1'], /require random/],
|
||||
[['--sequence', 'random', '--random-weights', 'success'], /expects behavior=weight/],
|
||||
[['--sequence', 'random', '--random-weights', 'random=1'], /concrete behavior/],
|
||||
[['--sequence', 'random', '--random-weights', 'success=1,success=2'], /duplicate/],
|
||||
[['--sequence', 'random', '--random-weights', 'success=nope'], /finite number/],
|
||||
])('rejects invalid argv %#', (argv, expected) => {
|
||||
expect(() => parseMockLlmCliArgs(argv)).toThrow(expected)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import InvariantService from '@deepseek-ai/dsh-invariants'
|
||||
import * as MockServerInvariant from '../src/invariant.ts'
|
||||
|
||||
describe('mock LLM server invariant companion', () => {
|
||||
it('registers its explained empty runtime invariant', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(InvariantService)
|
||||
const fiber = await ctx.plugin(MockServerInvariant)
|
||||
|
||||
expect(() => {
|
||||
ctx.invariants.register('@deepseek-ai/dsh-llm-mock-server', () => {})
|
||||
}).toThrow(/already registered/)
|
||||
await fiber.dispose()
|
||||
await ctx.fiber.dispose()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,312 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { MockLlmBehavior, MockLlmServer, MockLlmServerEvent } from '../src/index.ts'
|
||||
import { startMockLlmServer } from '../src/index.ts'
|
||||
|
||||
const running: MockLlmServer[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(running.splice(0).map(server => server.close()))
|
||||
})
|
||||
|
||||
async function start(
|
||||
sequence: readonly MockLlmBehavior[],
|
||||
options: Omit<Parameters<typeof startMockLlmServer>[0], 'sequence'> = {},
|
||||
): Promise<MockLlmServer> {
|
||||
const server = await startMockLlmServer({ sequence, ...options })
|
||||
running.push(server)
|
||||
return server
|
||||
}
|
||||
|
||||
function chat(
|
||||
server: MockLlmServer,
|
||||
options: { path?: string; key?: string; body?: string; signal?: AbortSignal } = {},
|
||||
): Promise<Response> {
|
||||
return fetch(`${server.baseURL}${options.path ?? '/v1/chat/completions'}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...options.key === undefined ? {} : { authorization: `Bearer ${options.key}` },
|
||||
},
|
||||
body: options.body ?? JSON.stringify({ model: 'mock', messages: [], stream: true }),
|
||||
...options.signal === undefined ? {} : { signal: options.signal },
|
||||
})
|
||||
}
|
||||
|
||||
describe('mock LLM server wire behaviors', () => {
|
||||
it('streams a complete text response and captures the request', async () => {
|
||||
const events: MockLlmServerEvent[] = []
|
||||
const server = await start(['success'], {
|
||||
apiKey: 'mock-key',
|
||||
successText: 'recovered',
|
||||
chunkSize: 3,
|
||||
onEvent: (event) => { events.push(event) },
|
||||
})
|
||||
|
||||
const response = await chat(server, { key: 'mock-key' })
|
||||
const body = await response.text()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(response.headers.get('content-type')).toContain('text/event-stream')
|
||||
expect(body).toContain('"content":"rec"')
|
||||
expect(body).toContain('"content":"ove"')
|
||||
expect(body).toContain('"content":"red"')
|
||||
expect(body).toContain('"finish_reason":"stop"')
|
||||
expect(body).toContain('data: [DONE]')
|
||||
expect(server.requests).toEqual([expect.objectContaining({
|
||||
attempt: 1,
|
||||
behavior: 'success',
|
||||
path: '/v1/chat/completions',
|
||||
body: { model: 'mock', messages: [], stream: true },
|
||||
chunksSent: 5,
|
||||
outcome: 'completed',
|
||||
})])
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: 'request',
|
||||
attempt: 1,
|
||||
scriptBehavior: 'success',
|
||||
behavior: 'success',
|
||||
path: '/v1/chat/completions',
|
||||
},
|
||||
{
|
||||
type: 'result',
|
||||
attempt: 1,
|
||||
scriptBehavior: 'success',
|
||||
behavior: 'success',
|
||||
outcome: 'completed',
|
||||
chunksSent: 5,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('supports root paths and intentionally ignores telemetry observer failures', async () => {
|
||||
const server = await start(['empty'], {
|
||||
onEvent() {
|
||||
throw new Error('observer failed')
|
||||
},
|
||||
})
|
||||
const response = await chat(server, { path: '/chat/completions' })
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.text()).toContain('data: [DONE]')
|
||||
expect(server.requests[0]).toMatchObject({ path: '/chat/completions', outcome: 'completed' })
|
||||
})
|
||||
|
||||
it.each([
|
||||
['empty_body', 0, ''] as const,
|
||||
['stream_eof', 1, '"role":"assistant"'] as const,
|
||||
['partial_eof', 1, 'discarded partial response'] as const,
|
||||
['malformed_json', 2, 'data: {not-json'] as const,
|
||||
['malformed_event', 2, '"choices":[null]'] as const,
|
||||
])('serves %s without inventing a terminal completion', async (behavior, chunks, marker) => {
|
||||
const server = await start([behavior], { chunkSize: 100 })
|
||||
const response = await chat(server)
|
||||
const body = await response.text()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(body).toContain(marker)
|
||||
if (behavior !== 'malformed_json' && behavior !== 'malformed_event') {
|
||||
expect(body).not.toContain('[DONE]')
|
||||
}
|
||||
expect(server.requests[0]).toMatchObject({ behavior, chunksSent: chunks, outcome: 'completed' })
|
||||
})
|
||||
|
||||
it.each([
|
||||
['connection_reset', false] as const,
|
||||
['stream_disconnect', true] as const,
|
||||
['partial_disconnect', true] as const,
|
||||
])('forces the %s transport boundary', async (behavior, receivesHeaders) => {
|
||||
const server = await start([behavior], { disconnectDelayMs: 20, partialText: 'half' })
|
||||
|
||||
let headersReceived = false
|
||||
await expect((async () => {
|
||||
const response = await chat(server)
|
||||
headersReceived = true
|
||||
await response.text()
|
||||
})()).rejects.toThrow()
|
||||
|
||||
expect(headersReceived).toBe(receivesHeaders)
|
||||
expect(server.requests[0]).toMatchObject({
|
||||
behavior,
|
||||
chunksSent: behavior === 'partial_disconnect' ? 1 : 0,
|
||||
outcome: 'reset',
|
||||
})
|
||||
})
|
||||
|
||||
it('holds a stalled stream until the client aborts and server close remains idempotent', async () => {
|
||||
const server = await start(['stall'])
|
||||
const controller = new AbortController()
|
||||
const response = await chat(server, { signal: controller.signal })
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(server.requests[0]).toMatchObject({ behavior: 'stall', outcome: 'stalled' })
|
||||
controller.abort()
|
||||
await expect(response.text()).rejects.toThrow()
|
||||
await server.close()
|
||||
await server.close()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['slow_success', 100] as const,
|
||||
['stream_disconnect', 100] as const,
|
||||
['partial_disconnect', 100] as const,
|
||||
])('records a client that closes during %s', async (behavior, delayMs) => {
|
||||
const server = await start([behavior], {
|
||||
chunkDelayMs: delayMs,
|
||||
disconnectDelayMs: delayMs,
|
||||
chunkSize: 1,
|
||||
})
|
||||
const controller = new AbortController()
|
||||
const response = await chat(server, { signal: controller.signal })
|
||||
controller.abort()
|
||||
await expect(response.text()).rejects.toThrow()
|
||||
await new Promise((resolve) => { setTimeout(resolve, 5) })
|
||||
|
||||
expect(server.requests[0]).toMatchObject({ behavior, outcome: 'client_closed' })
|
||||
})
|
||||
|
||||
it('emits reasoning, tool calls, max-token finishes, slow chunks, and a wrong content type', async () => {
|
||||
const server = await start([
|
||||
'reasoning_success',
|
||||
'tool_call_success',
|
||||
'max_tokens',
|
||||
'slow_success',
|
||||
'wrong_content_type',
|
||||
], {
|
||||
successText: 'answer',
|
||||
reasoningText: 'think',
|
||||
toolName: 'lookup',
|
||||
toolArguments: '{"id":7}',
|
||||
chunkDelayMs: 1,
|
||||
chunkSize: 2,
|
||||
})
|
||||
|
||||
const bodies: string[] = []
|
||||
const contentTypes: Array<string | null> = []
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const response = await chat(server)
|
||||
contentTypes.push(response.headers.get('content-type'))
|
||||
bodies.push(await response.text())
|
||||
}
|
||||
|
||||
expect(bodies[0]).toContain('"reasoning_content":"th"')
|
||||
expect(bodies[1]).toContain('"name":"lookup"')
|
||||
expect(bodies[1]).toContain('"arguments":"{\\"id"')
|
||||
expect(bodies[1]).toContain('"finish_reason":"tool_calls"')
|
||||
expect(bodies[2]).toContain('"finish_reason":"length"')
|
||||
expect(bodies[3]).toContain('"finish_reason":"stop"')
|
||||
expect(contentTypes[4]).toBe('application/json')
|
||||
expect(server.requests).toHaveLength(5)
|
||||
expect(server.requests.every(record => record.outcome === 'completed')).toBe(true)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['rate_limit', 429, 'mock rate limit'] as const,
|
||||
['server_error', 500, 'mock server error'] as const,
|
||||
['service_unavailable', 503, 'mock service unavailable'] as const,
|
||||
['auth_error', 401, 'mock authentication failed'] as const,
|
||||
['invalid_request', 400, 'mock invalid request'] as const,
|
||||
['context_overflow', 400, 'context_length_exceeded'] as const,
|
||||
['quota_exceeded', 429, 'insufficient_quota'] as const,
|
||||
])('serves %s as a structured HTTP error', async (behavior, status, marker) => {
|
||||
const server = await start([behavior], { retryAfterMs: 1_001, requestId: 'mock-request-1' })
|
||||
const response = await chat(server)
|
||||
const body = await response.text()
|
||||
|
||||
expect(response.status).toBe(status)
|
||||
expect(body).toContain(marker)
|
||||
expect(response.headers.get('x-request-id')).toBe('mock-request-1')
|
||||
if (behavior === 'rate_limit') expect(response.headers.get('retry-after')).toBe('2')
|
||||
else expect(response.headers.get('retry-after')).toBeNull()
|
||||
expect(server.requests[0]?.outcome).toBe('completed')
|
||||
})
|
||||
|
||||
it('fails loud on script exhaustion and can explicitly repeat the final behavior', async () => {
|
||||
const exhausted = await start(['success'], { successText: 'once' })
|
||||
await (await chat(exhausted)).text()
|
||||
const exhaustedResponse = await chat(exhausted)
|
||||
expect(exhaustedResponse.status).toBe(500)
|
||||
expect(await exhaustedResponse.text()).toContain('mock script exhausted')
|
||||
expect(exhausted.requests.map(record => record.behavior)).toEqual(['success', 'script_exhausted'])
|
||||
|
||||
const repeating = await start(['empty'], { repeatLast: true })
|
||||
await (await chat(repeating)).text()
|
||||
await (await chat(repeating)).text()
|
||||
expect(repeating.requests.map(record => record.behavior)).toEqual(['empty', 'empty'])
|
||||
})
|
||||
|
||||
it('selects weighted random behaviors reproducibly and reports the concrete choice', async () => {
|
||||
const options = {
|
||||
sequence: ['random'] as const,
|
||||
repeatLast: true,
|
||||
randomSeed: 42,
|
||||
randomWeights: { success: 1, empty: 1 },
|
||||
successText: 'random success',
|
||||
}
|
||||
const first = await startMockLlmServer(options)
|
||||
const second = await startMockLlmServer(options)
|
||||
running.push(first, second)
|
||||
|
||||
for (let attempt = 0; attempt < 12; attempt += 1) {
|
||||
await (await chat(first)).text()
|
||||
await (await chat(second)).text()
|
||||
}
|
||||
|
||||
const firstChoices = first.requests.map(record => record.behavior)
|
||||
expect(first.randomSeed).toBe(42)
|
||||
expect(second.randomSeed).toBe(42)
|
||||
expect(firstChoices).toEqual(second.requests.map(record => record.behavior))
|
||||
expect(new Set(firstChoices)).toEqual(new Set(['success', 'empty']))
|
||||
expect(first.requests.every(record => record.scriptBehavior === 'random')).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects invalid method, route, bearer token, and JSON without consuming the script', async () => {
|
||||
const server = await start(['success'], { apiKey: 'expected' })
|
||||
const method = await fetch(`${server.baseURL}/v1/chat/completions`)
|
||||
const route = await fetch(`${server.baseURL}/v1/other`, { method: 'POST', body: '{}' })
|
||||
const auth = await chat(server, { key: 'wrong' })
|
||||
const json = await chat(server, { key: 'expected', body: '{' })
|
||||
|
||||
expect(method.status).toBe(405)
|
||||
expect(method.headers.get('allow')).toBe('POST')
|
||||
expect(route.status).toBe(404)
|
||||
expect(auth.status).toBe(401)
|
||||
expect(json.status).toBe(400)
|
||||
expect(server.requests).toHaveLength(0)
|
||||
|
||||
const emptyRequest = await fetch(`${server.baseURL}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { authorization: 'Bearer expected' },
|
||||
})
|
||||
expect(emptyRequest.status).toBe(200)
|
||||
expect(server.requests[0]?.behavior).toBe('success')
|
||||
expect(server.requests[0]?.body).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('mock LLM server option validation', () => {
|
||||
it.each([
|
||||
[{ sequence: [] }, /sequence/],
|
||||
[{ sequence: ['success'], host: '' }, /host/],
|
||||
[{ sequence: ['success'], port: -1 }, /port/],
|
||||
[{ sequence: ['success'], port: 65_536 }, /port/],
|
||||
[{ sequence: ['success'], apiKey: '' }, /apiKey/],
|
||||
[{ sequence: ['success'], successText: '' }, /successText/],
|
||||
[{ sequence: ['success'], partialText: '' }, /partialText/],
|
||||
[{ sequence: ['success'], reasoningText: '' }, /reasoningText/],
|
||||
[{ sequence: ['success'], chunkSize: 0 }, /chunkSize/],
|
||||
[{ sequence: ['success'], chunkDelayMs: -1 }, /chunkDelayMs/],
|
||||
[{ sequence: ['success'], disconnectDelayMs: Number.POSITIVE_INFINITY }, /disconnectDelayMs/],
|
||||
[{ sequence: ['success'], retryAfterMs: 0 }, /retryAfterMs/],
|
||||
[{ sequence: ['success'], requestId: '' }, /requestId/],
|
||||
[{ sequence: ['success'], toolName: '' }, /toolName/],
|
||||
[{ sequence: ['success'], toolArguments: '{' }, /toolArguments/],
|
||||
[{ sequence: ['random'], randomSeed: -1 }, /randomSeed/],
|
||||
[{ sequence: ['random'], randomWeights: { random: 1 } }, /unknown concrete behavior/],
|
||||
[{ sequence: ['random'], randomWeights: { success: -1 } }, /non-negative/],
|
||||
[{ sequence: ['random'], randomWeights: { success: 0 } }, /positive weight/],
|
||||
] as const)('rejects invalid options %#', async (options, expected) => {
|
||||
await expect(startMockLlmServer(options as Parameters<typeof startMockLlmServer>[0]))
|
||||
.rejects.toThrow(expected)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user