Merge remote-tracking branch 'origin/master' into claude/unified-environment-credentials-c8841a
# Conflicts: # docs/config-catalog.md # packages/llm/llm-pi-ai/src/config.ts # packages/llm/llm-pi-ai/src/index.ts # packages/llm/llm-pi-ai/tests/sdk-options.spec.ts
This commit is contained in:
@@ -139,6 +139,22 @@ describe('PiAiAdapter provider routing', () => {
|
||||
expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }])
|
||||
})
|
||||
|
||||
it('names a route by its displayName, and by its own key once the profiles drop it', () => {
|
||||
const adapter = adapterOf({ 'acme-gateway': {
|
||||
displayName: 'Acme Gateway',
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://acme.test/v1',
|
||||
models: [{ id: 'acme-large' }],
|
||||
} })
|
||||
expect(adapter.providerInfo('acme-gateway')).toEqual({ id: 'acme-gateway', name: 'Acme Gateway' })
|
||||
|
||||
// The registry and the profiles can disagree for a moment: a refused
|
||||
// registration swap leaves the previous routes serving while resolution
|
||||
// has already moved on, so a selector may ask about a route the current
|
||||
// profiles no longer describe. It gets the key rather than nothing.
|
||||
expect(adapter.providerInfo('departed')).toEqual({ id: 'departed', name: 'departed' })
|
||||
})
|
||||
|
||||
it('reports unsupported stop sequences rather than silently ignoring them', async () => {
|
||||
const server = await mockServer([])
|
||||
const ctx = await harness(server.url)
|
||||
@@ -349,12 +365,11 @@ describe('provider profile lifecycle', () => {
|
||||
ReasoningEffortId('xhigh'),
|
||||
ReasoningEffortId('max'),
|
||||
])
|
||||
await expect(ctx.llm.resolveModelInfo('openai', 'gpt-4.1'))
|
||||
.resolves.toMatchObject({
|
||||
reasoning: {
|
||||
efforts: [{ id: ReasoningEffortId('off'), name: 'Off' }],
|
||||
},
|
||||
})
|
||||
// A catalog model without reasoning is the same case as a hand-declared
|
||||
// one: pi-ai reports the single level `off`, which translates to omitting
|
||||
// the reasoning option — exactly what naming no effort already does. The
|
||||
// capability is reported unavailable rather than offering that control.
|
||||
expect((await ctx.llm.resolveModelInfo('openai', 'gpt-4.1')).reasoning).toBeUndefined()
|
||||
})
|
||||
|
||||
it('uses a supported profile reasoning value as the model default and rejects an unsupported one', async () => {
|
||||
@@ -418,12 +433,14 @@ describe('provider profile lifecycle', () => {
|
||||
expect(server.requests).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('validates empty, unknown, legacy-shaped, and explicitly blank profiles', () => {
|
||||
it('validates empty, underspecified, legacy-shaped, and explicitly blank profiles', () => {
|
||||
// Empty and omitted dicts are the dormant zero-route posture, not errors.
|
||||
expect(resolveProfiles({}).size).toBe(0)
|
||||
expect(resolveProfiles(undefined).size).toBe(0)
|
||||
expect(() => resolveProfiles({ '': {} })).toThrow(/non-empty/)
|
||||
expect(() => resolveProfiles({ 'not-real': {} })).toThrow(/unknown/)
|
||||
// A route the installed catalog does not ship is allowed, but it has no
|
||||
// defaults to fall back on: it must describe its own models.
|
||||
expect(() => resolveProfiles({ 'not-real': {} })).toThrow(/resolves no models/)
|
||||
// The pre-release array shape and its per-profile provider field fail
|
||||
// loud with migration directions instead of half-working.
|
||||
expect(() => resolveProfiles([{ provider: 'openai' }] as never)).toThrow(/dict keyed by provider/)
|
||||
|
||||
@@ -0,0 +1,583 @@
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService, { createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import type { StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import SettingsLocal from '@deepseek-ai/dsh-settings-local'
|
||||
import { settingsNamespace } from '@deepseek-ai/dsh-settings'
|
||||
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import { PiAiAdapter } from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import { getBuiltinModels } from '@earendil-works/pi-ai/providers/all'
|
||||
import { createModels } from '@earendil-works/pi-ai'
|
||||
import type { Api, Model, Provider } from '@earendil-works/pi-ai'
|
||||
import { resolveProfiles } from '../src/config.ts'
|
||||
import { buildProvider, supportedProtocols } from '../src/provider.ts'
|
||||
import { assemble } from './assemble.ts'
|
||||
import { closeMockServers, mockServer, textEvents } from './mock-server.ts'
|
||||
|
||||
const homes: string[] = []
|
||||
|
||||
// Routes name their credential by reference; the value lives in the
|
||||
// environment, which is the layer the adapter falls back to without a
|
||||
// mounted credentials seam.
|
||||
const KEY_ENV = 'PI_TEST_KEY'
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubEnv(KEY_ENV, 'test-key')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs()
|
||||
await closeMockServers()
|
||||
await Promise.all(homes.splice(0).map(dir => rm(dir, { recursive: true, force: true })))
|
||||
})
|
||||
|
||||
/** A throwaway $DSH_HOME with an empty settings document. */
|
||||
async function home(): Promise<string> {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'dsh-pi-catalog-'))
|
||||
homes.push(dir)
|
||||
await writeFile(join(dir, 'settings.yaml'), '')
|
||||
return dir
|
||||
}
|
||||
|
||||
/** The dormant composition plus a real settings service, as the product mounts it. */
|
||||
async function bootWithSettings(dir: string, config: LlmPiAi.Config): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(SettingsLocal, { path: join(dir, 'settings.yaml'), watch: false })
|
||||
await ctx.plugin(LlmPiAi, config)
|
||||
return ctx
|
||||
}
|
||||
|
||||
/** A complete hand-declared route: nothing about it exists in pi-ai's catalog. */
|
||||
function gateway(baseURL: string, overrides: Record<string, unknown> = {}): LlmPiAi.Config {
|
||||
return {
|
||||
providers: {
|
||||
'acme-gateway': {
|
||||
apiKeyEnv: KEY_ENV,
|
||||
displayName: 'Acme Gateway',
|
||||
api: 'openai-completions',
|
||||
baseURL,
|
||||
models: [{ id: 'acme-large', name: 'Acme Large', contextWindow: 65_536, maxTokens: 4096 }],
|
||||
...overrides,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function harness(config: LlmPiAi.Config): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(LlmPiAi, config)
|
||||
return ctx
|
||||
}
|
||||
|
||||
describe('hand-declared providers', () => {
|
||||
it('serves a route pi-ai has never heard of from its own declaration', async () => {
|
||||
const server = await mockServer([{ events: textEvents }])
|
||||
const ctx = await harness(gateway(`${server.url}/v1`))
|
||||
|
||||
const result = await assemble(ctx, {
|
||||
provider: 'acme-gateway',
|
||||
model: 'acme-large',
|
||||
messages: [createUserMessage({
|
||||
content: [{ type: 'text', text: 'hi' }],
|
||||
source: { kind: 'plugin', plugin: 'test' },
|
||||
})],
|
||||
})
|
||||
|
||||
expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }])
|
||||
expect(result.finish).toEqual({ kind: 'stop' })
|
||||
expect(server.paths).toEqual(['/v1/chat/completions'])
|
||||
// The reference resolved through the environment and reached the wire.
|
||||
expect(server.headers[0]?.authorization).toBe('Bearer test-key')
|
||||
})
|
||||
|
||||
it('lists and resolves the declared models rather than a catalog', async () => {
|
||||
const server = await mockServer([])
|
||||
const ctx = await harness(gateway(`${server.url}/v1`))
|
||||
|
||||
expect(await ctx.llm.listModels('acme-gateway')).toEqual([
|
||||
{ provider: 'acme-gateway', id: 'acme-large', name: 'Acme Large' },
|
||||
])
|
||||
const info = await ctx.llm.resolveModelInfo('acme-gateway', 'acme-large')
|
||||
expect(info).toMatchObject({
|
||||
provider: 'acme-gateway',
|
||||
id: 'acme-large',
|
||||
name: 'Acme Large',
|
||||
context: { contextWindow: 65_536 },
|
||||
defaultMaxTokens: 4096,
|
||||
})
|
||||
})
|
||||
|
||||
it('offers no reasoning control it could not honour', async () => {
|
||||
const server = await mockServer([])
|
||||
const ctx = await harness(gateway(`${server.url}/v1`))
|
||||
|
||||
// pi-ai reports a model with no reasoning metadata as supporting the single
|
||||
// level `off`, but `off` is translated to *omitting* the reasoning option —
|
||||
// byte-for-byte the same request as naming no effort — so a provider whose
|
||||
// own default is to think would keep thinking with `off` selected. The
|
||||
// capability is reported unavailable instead of offering that control.
|
||||
expect((await ctx.llm.resolveModelInfo('acme-gateway', 'acme-large')).reasoning).toBeUndefined()
|
||||
|
||||
// A catalog route is unaffected: its models carry the metadata that makes
|
||||
// `off` actually disable thinking.
|
||||
const withCatalog = await harness({ providers: { deepseek: { baseURL: server.url } } })
|
||||
const [catalogModel] = getBuiltinModels('deepseek')
|
||||
if (catalogModel === undefined) throw new Error('the installed catalog ships no deepseek model')
|
||||
expect((await withCatalog.llm.resolveModelInfo('deepseek', catalogModel.id)).reasoning?.efforts.map(e => e.id))
|
||||
.toContain('off')
|
||||
})
|
||||
|
||||
it('joins the configurable-provider directory so a settings surface can reach it', async () => {
|
||||
const server = await mockServer([])
|
||||
const ctx = await harness(gateway(`${server.url}/v1`))
|
||||
|
||||
expect(ctx.llm.listConfigurableProviders()).toContainEqual({
|
||||
provider: 'acme-gateway',
|
||||
displayName: 'Acme Gateway',
|
||||
settingsNs: 'llm-pi-ai',
|
||||
settingsPath: ['providers', 'acme-gateway'],
|
||||
})
|
||||
})
|
||||
|
||||
it('sizes a model the catalog cannot describe from the route\u2019s own fallbacks', () => {
|
||||
const resolved = resolveProfiles({
|
||||
'acme-gateway': {
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://acme.test',
|
||||
// A listing endpoint that discloses nothing but ids still yields a
|
||||
// serviceable route.
|
||||
models: [{ id: 'bare' }, { id: 'sized', contextWindow: 8192, maxTokens: 512 }],
|
||||
},
|
||||
'tuned-gateway': {
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://tuned.test',
|
||||
defaultContextWindow: 4096,
|
||||
defaultMaxTokens: 256,
|
||||
models: [{ id: 'bare' }],
|
||||
},
|
||||
})
|
||||
const modelsOf = (route: string): readonly { id: string; contextWindow: number; maxTokens: number }[] =>
|
||||
resolved.get(route)?.piProvider.getModels() ?? []
|
||||
|
||||
expect(modelsOf('acme-gateway')).toMatchObject([
|
||||
{ id: 'bare', contextWindow: 262_144, maxTokens: 32_768 },
|
||||
{ id: 'sized', contextWindow: 8192, maxTokens: 512 },
|
||||
])
|
||||
// The fallback is a guess, so a deployment whose gateway serves smaller
|
||||
// models corrects it once for the whole route.
|
||||
expect(modelsOf('tuned-gateway')).toMatchObject([{ id: 'bare', contextWindow: 4096, maxTokens: 256 }])
|
||||
// Only an explicitly configured cap is a request default; a fallback is
|
||||
// the model's capability and stops there.
|
||||
expect(resolved.get('acme-gateway')?.configuredMaxTokens.get('bare')).toBeUndefined()
|
||||
expect(resolved.get('acme-gateway')?.configuredMaxTokens.get('sized')).toBe(512)
|
||||
})
|
||||
|
||||
it('rejects a model the route cannot identify', () => {
|
||||
const declare = (model: LlmPiAi.PiAiModelProfile): (() => unknown) =>
|
||||
() => resolveProfiles({ 'acme-gateway': { api: 'openai-completions', baseURL: 'https://acme.test', models: [model] } })
|
||||
|
||||
expect(declare({ id: '' })).toThrow(/empty id/)
|
||||
expect(() => resolveProfiles({
|
||||
'acme-gateway': {
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://acme.test',
|
||||
models: [{ id: 'dup', contextWindow: 1, maxTokens: 1 }, { id: 'dup', contextWindow: 2, maxTokens: 2 }],
|
||||
},
|
||||
})).toThrow(/more than once/)
|
||||
})
|
||||
|
||||
it('rejects a declaration that names no wire protocol or endpoint', () => {
|
||||
expect(() => resolveProfiles({
|
||||
'acme-gateway': { baseURL: 'https://acme.test', models: [{ id: 'm', contextWindow: 1, maxTokens: 1 }] },
|
||||
})).toThrow(/needs an api/)
|
||||
expect(() => resolveProfiles({
|
||||
'acme-gateway': { api: 'openai-completions', models: [{ id: 'm', contextWindow: 1, maxTokens: 1 }] },
|
||||
})).toThrow(/needs a baseURL/)
|
||||
})
|
||||
|
||||
it.each(['bedrock-converse-stream', 'google-vertex', 'azure-openai-responses', 'openai-codex-responses'])(
|
||||
'refuses %s, whose authentication a profile cannot express',
|
||||
(api) => {
|
||||
// These need SigV4 credentials and a region, a project plus ADC, provider
|
||||
// environment and an api-version, or OAuth — none of which a key, an
|
||||
// endpoint, and headers can carry, so a route naming one would be built
|
||||
// unable to authenticate.
|
||||
expect(supportedProtocols()).not.toContain(api)
|
||||
expect(() => buildProvider({ provider: 'acme-gateway', displayName: 'Acme', api, models: [], namesCredential: true }))
|
||||
.toThrow(/cannot serve; supported protocols are/)
|
||||
},
|
||||
)
|
||||
|
||||
it('rejects a protocol this build cannot serve, and a route that names none', () => {
|
||||
const spec = { provider: 'acme-gateway', displayName: 'Acme Gateway', models: [], namesCredential: true }
|
||||
expect(() => buildProvider({ ...spec, api: 'quantum-telepathy' }))
|
||||
.toThrow(/cannot serve; supported protocols are/)
|
||||
expect(() => buildProvider(spec)).toThrow(/cannot serve; supported protocols are/)
|
||||
})
|
||||
|
||||
it('leaves an unauthenticated route to its protocol rather than inventing a credential', async () => {
|
||||
const server = await mockServer([{ events: textEvents }])
|
||||
// Naming no credential is the deliberately unauthenticated posture — a
|
||||
// named reference that resolved to nothing would have failed with
|
||||
// MISSING_CREDENTIAL long before this point. The route resolves as
|
||||
// configured and the protocol decides: pi-ai's OpenAI-compatible
|
||||
// implementation wants a key or an Authorization header of its own, and
|
||||
// says so instead of the harness guessing a placeholder.
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
'local-llm': {
|
||||
api: 'openai-completions',
|
||||
baseURL: `${server.url}/v1`,
|
||||
models: [{ id: 'qwen3', contextWindow: 32_768, maxTokens: 2048 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const result = await assemble(ctx, { provider: 'local-llm', model: 'qwen3', messages: [] })
|
||||
expect(result.finish).toMatchObject({
|
||||
kind: 'error',
|
||||
failure: { message: 'No API key for provider: local-llm' },
|
||||
})
|
||||
expect(server.requests).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('authenticates an unauthenticated route through a configured header', async () => {
|
||||
const server = await mockServer([{ events: textEvents }])
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
'local-llm': {
|
||||
api: 'openai-completions',
|
||||
baseURL: `${server.url}/v1`,
|
||||
headers: { Authorization: 'Bearer local' },
|
||||
models: [{ id: 'qwen3', contextWindow: 32_768, maxTokens: 2048 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const result = await assemble(ctx, { provider: 'local-llm', model: 'qwen3', messages: [] })
|
||||
expect(result.finish).toEqual({ kind: 'stop' })
|
||||
expect(server.headers[0]?.authorization).toBe('Bearer local')
|
||||
})
|
||||
|
||||
it('rejects a capacity that is not a positive integer', () => {
|
||||
const declare = (model: LlmPiAi.PiAiModelProfile): (() => unknown) =>
|
||||
() => resolveProfiles({ 'acme-gateway': { api: 'openai-completions', baseURL: 'https://acme.test', models: [model] } })
|
||||
|
||||
expect(declare({ id: 'm', contextWindow: 0, maxTokens: 1 })).toThrow(/contextWindow must be a positive integer/)
|
||||
expect(declare({ id: 'm', contextWindow: 1.5, maxTokens: 1 })).toThrow(/contextWindow must be a positive integer/)
|
||||
expect(declare({ id: 'm', contextWindow: 1, maxTokens: 0 })).toThrow(/maxTokens must be a positive integer/)
|
||||
expect(declare({ id: 'm', contextWindow: 1, maxTokens: 1.5 })).toThrow(/maxTokens must be a positive integer/)
|
||||
})
|
||||
|
||||
it('names the route key when no displayName is configured', () => {
|
||||
const resolved = resolveProfiles({
|
||||
'acme-gateway': {
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://acme.test',
|
||||
models: [{ id: 'm', contextWindow: 1, maxTokens: 1 }],
|
||||
},
|
||||
})
|
||||
expect(resolved.get('acme-gateway')?.displayName).toBe('acme-gateway')
|
||||
expect(() => resolveProfiles({ 'acme-gateway': { displayName: '' } })).toThrow(/empty displayName/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('catalog routes with per-model configuration', () => {
|
||||
it('serves the installed catalog untouched when the profile lists no models', async () => {
|
||||
const server = await mockServer([])
|
||||
const ctx = await harness({ providers: { deepseek: { baseURL: server.url } } })
|
||||
|
||||
const listed = await ctx.llm.listModels('deepseek')
|
||||
expect(listed.map(model => model.id).sort())
|
||||
.toEqual(getBuiltinModels('deepseek').map(model => model.id).sort())
|
||||
})
|
||||
|
||||
it('overrides one catalog model field and defaults the rest from the catalog', async () => {
|
||||
const server = await mockServer([])
|
||||
const [catalogModel] = getBuiltinModels('deepseek')
|
||||
if (catalogModel === undefined) throw new Error('the installed catalog ships no deepseek model')
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
deepseek: {
|
||||
baseURL: server.url,
|
||||
models: [{ id: catalogModel.id, contextWindow: 4096 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const info = await ctx.llm.resolveModelInfo('deepseek', catalogModel.id)
|
||||
// The configured field wins and the name still comes from the catalog. The
|
||||
// catalog's own output cap is the model's capability, not a cap anyone
|
||||
// chose, so it must not arrive as the request default.
|
||||
expect(info.context).toEqual({ contextWindow: 4096 })
|
||||
expect(info.name).toBe(catalogModel.name)
|
||||
expect(info.defaultMaxTokens).toBeUndefined()
|
||||
// An explicit list replaces the catalog rather than adding to it.
|
||||
expect((await ctx.llm.listModels('deepseek')).map(model => model.id)).toEqual([catalogModel.id])
|
||||
})
|
||||
|
||||
it('materializes a request default only from a configured output cap', async () => {
|
||||
const server = await mockServer([])
|
||||
const [catalogModel] = getBuiltinModels('deepseek')
|
||||
if (catalogModel === undefined) throw new Error('the installed catalog ships no deepseek model')
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
deepseek: {
|
||||
baseURL: server.url,
|
||||
models: [{ id: catalogModel.id, maxTokens: 4096 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// Configuring the cap is the deployment choosing one, so it becomes the
|
||||
// default the seam materializes into requests that name none.
|
||||
expect((await ctx.llm.resolveModelInfo('deepseek', catalogModel.id)).defaultMaxTokens).toBe(4096)
|
||||
})
|
||||
|
||||
it('adds a model the installed catalog does not describe to a catalog route', async () => {
|
||||
const server = await mockServer([{ events: textEvents }])
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
deepseek: {
|
||||
apiKeyEnv: KEY_ENV,
|
||||
baseURL: `${server.url}/v1`,
|
||||
models: [{ id: 'deepseek-preview', contextWindow: 200_000, maxTokens: 8192 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const result = await assemble(ctx, { provider: 'deepseek', model: 'deepseek-preview', messages: [] })
|
||||
expect(result.finish).toEqual({ kind: 'stop' })
|
||||
// The catalog route keeps its catalog protocol, so the new model reaches
|
||||
// the same endpoint shape the shipped models use.
|
||||
expect(server.paths).toEqual(['/v1/chat/completions'])
|
||||
})
|
||||
|
||||
it('fails an unconfigured model id before any provider request', async () => {
|
||||
const server = await mockServer([])
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
deepseek: { baseURL: server.url, models: [{ id: 'deepseek-preview', contextWindow: 1, maxTokens: 1 }] },
|
||||
},
|
||||
})
|
||||
|
||||
const result = await assemble(ctx, { provider: 'deepseek', model: 'not-configured', messages: [] })
|
||||
|
||||
expect(result.finish).toMatchObject({ kind: 'error', failure: { code: 'UNKNOWN_MODEL' } })
|
||||
expect(server.requests).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('preserves catalog-only model metadata the profile cannot express', () => {
|
||||
// Some catalog models carry provider-required request headers; overriding a
|
||||
// capacity must not drop them, because configuration has no way to restate
|
||||
// them.
|
||||
const headered = (getBuiltinModels('nvidia') as { id: string; headers?: unknown }[])
|
||||
.find(model => model.headers !== undefined)
|
||||
if (headered === undefined) throw new Error('the installed catalog ships no nvidia model with headers')
|
||||
|
||||
const resolved = resolveProfiles({
|
||||
nvidia: { models: [{ id: headered.id, contextWindow: 4096 }] },
|
||||
})
|
||||
const [model] = resolved.get('nvidia')?.piProvider.getModels() ?? []
|
||||
expect(model?.headers).toEqual(headered.headers)
|
||||
expect(model?.contextWindow).toBe(4096)
|
||||
})
|
||||
|
||||
it('delegates both stream methods back to the reused catalog provider', async () => {
|
||||
const server = await mockServer([{ events: textEvents }, { events: textEvents }])
|
||||
const resolved = resolveProfiles({ deepseek: { baseURL: `${server.url}/v1` } })
|
||||
const built = resolved.get('deepseek')?.piProvider
|
||||
if (built === undefined) throw new Error('the deepseek route built no provider')
|
||||
const [model] = built.getModels()
|
||||
if (model === undefined) throw new Error('the deepseek route resolved no models')
|
||||
const context = { messages: [{ role: 'user' as const, content: 'hi', timestamp: 0 }] }
|
||||
|
||||
// `stream` is interface-required and unused by the harness adapter, which
|
||||
// only calls `streamSimple`; both must still reach the catalog provider.
|
||||
for await (const _event of built.stream(model, context, { apiKey: 'k' })) { /* drain */ }
|
||||
for await (const _event of built.streamSimple(model, context, { apiKey: 'k' })) { /* drain */ }
|
||||
|
||||
expect(server.paths).toEqual(['/v1/chat/completions', '/v1/chat/completions'])
|
||||
})
|
||||
|
||||
it('keeps each model its own endpoint when the catalog route declares none', () => {
|
||||
// `opencode` ships no provider-level endpoint: the address lives on every
|
||||
// catalog model, so the route resolves without any configured baseURL.
|
||||
const resolved = resolveProfiles({ opencode: {} })
|
||||
const models = resolved.get('opencode')?.piProvider.getModels() ?? []
|
||||
expect(models.length).toBeGreaterThan(0)
|
||||
expect(models.every(model => model.baseUrl.length > 0)).toBe(true)
|
||||
expect(resolved.get('opencode')?.piProvider.baseUrl).toBeUndefined()
|
||||
})
|
||||
|
||||
it('repoints a catalog route at another wire protocol without restating its endpoint', () => {
|
||||
const resolved = resolveProfiles({ openai: { api: 'openai-completions' } })
|
||||
const models = resolved.get('openai')?.piProvider.getModels() ?? []
|
||||
// The protocol changes for the whole route; each model keeps the catalog
|
||||
// endpoint it already had.
|
||||
expect(models.every(model => model.api === 'openai-completions')).toBe(true)
|
||||
expect(models.every(model => model.baseUrl === 'https://api.openai.com/v1')).toBe(true)
|
||||
})
|
||||
|
||||
it('repoints a catalog route at another wire protocol', async () => {
|
||||
const server = await mockServer([{ events: textEvents }])
|
||||
const ctx = await harness({
|
||||
providers: {
|
||||
// openai's catalog models speak the Responses API; naming the protocol
|
||||
// explicitly moves the whole route onto Chat Completions.
|
||||
openai: {
|
||||
apiKeyEnv: KEY_ENV,
|
||||
api: 'openai-completions',
|
||||
baseURL: `${server.url}/v1`,
|
||||
models: [{ id: 'gpt-4.1', contextWindow: 100_000, maxTokens: 4096 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] })
|
||||
expect(server.paths).toEqual(['/v1/chat/completions'])
|
||||
})
|
||||
|
||||
it('keeps the catalog provider’s own auth when the route repoints its protocol', () => {
|
||||
// Which environment a provider reads is a property of the provider, not of
|
||||
// the wire format its models speak: naming an api must not cost a profile
|
||||
// its provider-native discovery.
|
||||
const resolved = resolveProfiles({ openai: { api: 'openai-completions' } })
|
||||
expect(resolved.get('openai')?.piProvider.auth.apiKey?.name).toBe('OpenAI API key')
|
||||
})
|
||||
|
||||
it('lets an OAuth-only catalog route authenticate with the key its profile names', async () => {
|
||||
// pi-ai honours a request's `apiKey` override only when the provider
|
||||
// declares an api-key method. `openai-codex` ships OAuth alone, so without
|
||||
// the harness method beside it the route refuses its own configured key as
|
||||
// `Provider is not configured` before any request goes out.
|
||||
const resolved = resolveProfiles({ 'openai-codex': { apiKeyEnv: 'CODEX_TOKEN' } })
|
||||
const provider = resolved.get('openai-codex')?.piProvider
|
||||
expect(provider?.auth.oauth).toBeDefined()
|
||||
const models = createModels()
|
||||
models.setProvider(provider as Provider)
|
||||
const model = provider?.getModels()[0] as Model<Api>
|
||||
const auth = await models.getAuth(model, { apiKey: 'codex-token' })
|
||||
expect(auth?.auth.apiKey).toBe('codex-token')
|
||||
})
|
||||
|
||||
it('leaves an OAuth-only catalog route unconfigured when its profile names no key', () => {
|
||||
// Nothing to add: this adapter resolves credentials through its own seam
|
||||
// and holds no OAuth store, so declaring the provider configured would
|
||||
// trade a truthful refusal for an endpoint's 401.
|
||||
const resolved = resolveProfiles({ 'openai-codex': {} })
|
||||
expect(resolved.get('openai-codex')?.piProvider.auth.apiKey).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolution snapshots', () => {
|
||||
it('finishes an in-flight request under the configuration it started with', async () => {
|
||||
const server = await mockServer([{ events: textEvents }])
|
||||
let current = resolveProfiles({ deepseek: { baseURL: `${server.url}/v1` } })
|
||||
let release: () => void = () => {}
|
||||
const held = new Promise<void>((resolve) => { release = resolve })
|
||||
const adapter = new PiAiAdapter({
|
||||
profiles: () => current,
|
||||
// Credential resolution is the real await inside a stream call, and the
|
||||
// window a configuration change has to land in.
|
||||
resolveApiKey: async () => { await held; return 'k' },
|
||||
})
|
||||
|
||||
const chunks: StreamChunk[] = []
|
||||
const inFlight = (async () => {
|
||||
for await (const chunk of adapter.stream({
|
||||
provider: 'deepseek',
|
||||
model: 'deepseek-v4-flash',
|
||||
messages: [],
|
||||
})) chunks.push(chunk)
|
||||
})()
|
||||
|
||||
// The route set changes while the request waits, and something else reads
|
||||
// the adapter meanwhile, which is what would rebuild a shared collection.
|
||||
current = resolveProfiles({ openai: { baseURL: `${server.url}/v1` } })
|
||||
await expect(adapter.listModels('openai')).resolves.not.toHaveLength(0)
|
||||
release()
|
||||
await inFlight
|
||||
|
||||
// The in-flight request keeps its own snapshot: it reaches the endpoint it
|
||||
// resolved against instead of failing on a provider that no longer exists.
|
||||
expect(chunks.at(-1)).toMatchObject({ type: 'finish', reason: { kind: 'stop' } })
|
||||
expect(server.paths).toEqual(['/v1/chat/completions'])
|
||||
})
|
||||
|
||||
it('serves the next request from the new configuration', async () => {
|
||||
const first = await mockServer([{ events: textEvents }])
|
||||
const second = await mockServer([{ events: textEvents }])
|
||||
let current = resolveProfiles({ deepseek: { baseURL: `${first.url}/v1` } })
|
||||
const adapter = new PiAiAdapter({ profiles: () => current, resolveApiKey: () => Promise.resolve('k') })
|
||||
const drain = async (): Promise<void> => {
|
||||
for await (const _chunk of adapter.stream({
|
||||
provider: 'deepseek', model: 'deepseek-v4-flash', messages: [],
|
||||
})) { /* drain */ }
|
||||
}
|
||||
|
||||
await drain()
|
||||
current = resolveProfiles({ deepseek: { baseURL: `${second.url}/v1` } })
|
||||
await drain()
|
||||
|
||||
expect(first.paths).toHaveLength(1)
|
||||
expect(second.paths).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('configurable-provider directory', () => {
|
||||
it('keeps the previous directory when a route collides with another adapter family', async () => {
|
||||
const dir = await home()
|
||||
const ctx = await bootWithSettings(dir, {})
|
||||
// Another adapter family owns this route id, exactly as llm-deepseek does.
|
||||
ctx.llm.registerConfigurableProviders([
|
||||
{ provider: 'deepseek-official', displayName: 'DeepSeek', settingsNs: 'llm-deepseek', settingsPath: [] },
|
||||
])
|
||||
const before = ctx.llm.listConfigurableProviders().length
|
||||
expect(before).toBeGreaterThan(30)
|
||||
|
||||
await ctx.settings.update(settingsNamespace('llm-pi-ai'), {
|
||||
providers: {
|
||||
'deepseek-official': {
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://acme.test/v1',
|
||||
models: [{ id: 'm', contextWindow: 1, maxTokens: 1 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// The refused swap costs a diagnostic, not the directory: every entry the
|
||||
// page needs is still declared.
|
||||
expect(ctx.llm.listConfigurableProviders()).toHaveLength(before)
|
||||
expect(ctx.llm.listConfigurableProviders().find(entry => entry.provider === 'deepseek-official')?.settingsNs)
|
||||
.toBe('llm-deepseek')
|
||||
})
|
||||
|
||||
it('replaces its entries atomically as declared routes come and go', async () => {
|
||||
const dir = await home()
|
||||
const ctx = await bootWithSettings(dir, {})
|
||||
const catalogOnly = ctx.llm.listConfigurableProviders().length
|
||||
|
||||
await ctx.settings.update(settingsNamespace('llm-pi-ai'), {
|
||||
providers: {
|
||||
'acme-gateway': {
|
||||
displayName: 'Acme Gateway',
|
||||
api: 'openai-completions',
|
||||
baseURL: 'https://acme.test/v1',
|
||||
models: [{ id: 'm', contextWindow: 1, maxTokens: 1 }],
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(ctx.llm.listConfigurableProviders()).toHaveLength(catalogOnly + 1)
|
||||
expect(ctx.llm.listConfigurableProviders().find(entry => entry.provider === 'acme-gateway')?.displayName)
|
||||
.toBe('Acme Gateway')
|
||||
|
||||
await ctx.settings.replace(settingsNamespace('llm-pi-ai'), {})
|
||||
expect(ctx.llm.listConfigurableProviders()).toHaveLength(catalogOnly)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,313 @@
|
||||
import { createServer } from 'node:http'
|
||||
import type { IncomingMessage, Server, ServerResponse } from 'node:http'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService, { userAgent } from '@deepseek-ai/dsh-llm'
|
||||
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import { getBuiltinModels } from '@earendil-works/pi-ai/providers/all'
|
||||
import { discoverModels } from '../src/discovery.ts'
|
||||
|
||||
const servers: Server[] = []
|
||||
/** Credential variables a test set, cleared so the next one starts unset. */
|
||||
const touchedEnv: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const name of touchedEnv.splice(0)) Reflect.deleteProperty(process.env, name)
|
||||
await Promise.all(servers.splice(0).map(server => new Promise(resolve => server.close(resolve))))
|
||||
})
|
||||
|
||||
interface ListingServer {
|
||||
url: string
|
||||
paths: string[]
|
||||
headers: IncomingMessage['headers'][]
|
||||
}
|
||||
|
||||
/**
|
||||
* A stand-in provider that answers one scripted `GET /models`. `chunks` writes
|
||||
* without a declared length, which is how a real streamed reply arrives.
|
||||
*/
|
||||
async function listingServer(behavior: {
|
||||
status?: number
|
||||
body?: string
|
||||
chunks?: string[]
|
||||
holdOpenMs?: number
|
||||
}): Promise<ListingServer> {
|
||||
const paths: string[] = []
|
||||
const headers: IncomingMessage['headers'][] = []
|
||||
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
|
||||
paths.push(request.url ?? '')
|
||||
headers.push(request.headers)
|
||||
if (behavior.chunks !== undefined) {
|
||||
// No declared length: the ceiling has to hold on what is read.
|
||||
response.writeHead(behavior.status ?? 200, { 'content-type': 'application/json' })
|
||||
for (const chunk of behavior.chunks) response.write(chunk)
|
||||
if (behavior.holdOpenMs === undefined) { response.end(); return }
|
||||
// Left open so a caller's cancellation lands while the body is still
|
||||
// being read rather than after it completed.
|
||||
setTimeout(() => { response.end() }, behavior.holdOpenMs)
|
||||
return
|
||||
}
|
||||
const body = behavior.body ?? '{}'
|
||||
response.writeHead(behavior.status ?? 200, {
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(Buffer.byteLength(body)),
|
||||
})
|
||||
response.end(body)
|
||||
})
|
||||
servers.push(server)
|
||||
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
|
||||
const address = server.address()
|
||||
if (address === null || typeof address === 'string') throw new Error('no port')
|
||||
return { url: `http://127.0.0.1:${address.port}`, paths, headers }
|
||||
}
|
||||
|
||||
/** A bare dormant mount: discovery is offered whether or not a route exists. */
|
||||
async function harness(): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(LlmPiAi, {})
|
||||
return ctx
|
||||
}
|
||||
|
||||
describe('catalog-route model discovery', () => {
|
||||
it('answers from the installed registry, with capacities and no network call', async () => {
|
||||
const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'from-the-endpoint' }] }) })
|
||||
const ctx = await harness()
|
||||
|
||||
const models = await ctx.llm.discoverModels('llm-pi-ai', { provider: 'deepseek', baseURL: server.url })
|
||||
|
||||
// pi-ai's own registry is the authority for its own providers, and it
|
||||
// carries what a listing endpoint would not disclose.
|
||||
expect(models.map(model => model.id).sort())
|
||||
.toEqual(getBuiltinModels('deepseek').map(model => model.id).sort())
|
||||
expect(models.every(model => (model.contextWindow ?? 0) > 0 && (model.maxTokens ?? 0) > 0)).toBe(true)
|
||||
expect(server.paths).toEqual([])
|
||||
})
|
||||
|
||||
it('needs no endpoint for a route the catalog describes', async () => {
|
||||
const ctx = await harness()
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'deepseek' })).resolves.not.toHaveLength(0)
|
||||
})
|
||||
|
||||
it('says where a route the catalog does not describe must get its models', async () => {
|
||||
const ctx = await harness()
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'acme-gateway' }))
|
||||
.rejects.toThrow(/ships no catalog for provider "acme-gateway".*set a baseURL/s)
|
||||
// A form that cleared the field says the same thing as one that never had it.
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'acme-gateway', baseURL: '' }))
|
||||
.rejects.toThrow(/set a baseURL/)
|
||||
// The seam refuses a request naming neither, so the module's own guard for
|
||||
// that shape is only reachable by calling it directly.
|
||||
await expect(discoverModels({})).rejects.toThrow(/set a baseURL/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('draft-provider model discovery', () => {
|
||||
it('reads an OpenAI-compatible listing and keeps the capacities it discloses', async () => {
|
||||
const server = await listingServer({
|
||||
body: JSON.stringify({
|
||||
data: [
|
||||
{ id: 'acme-large', display_name: 'Acme Large', context_length: 65_536, max_output_tokens: 4096 },
|
||||
{ id: 'acme-small' },
|
||||
],
|
||||
}),
|
||||
})
|
||||
const ctx = await harness()
|
||||
|
||||
const models = await ctx.llm.discoverModels('llm-pi-ai', { baseURL: `${server.url}/v1`, apiKey: 'probe-key' })
|
||||
|
||||
expect(models).toEqual([
|
||||
{ id: 'acme-large', name: 'Acme Large', contextWindow: 65_536, maxTokens: 4096 },
|
||||
{ id: 'acme-small' },
|
||||
])
|
||||
expect(server.paths).toEqual(['/v1/models'])
|
||||
expect(server.headers[0]?.authorization).toBe('Bearer probe-key')
|
||||
expect(server.headers[0]?.['user-agent']).toBe(userAgent())
|
||||
})
|
||||
|
||||
it('keeps a deployment path instead of resolving it away', async () => {
|
||||
const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'm' }] }) })
|
||||
const ctx = await harness()
|
||||
|
||||
await ctx.llm.discoverModels('llm-pi-ai', { baseURL: `${server.url}/openai/v1/` })
|
||||
|
||||
expect(server.paths).toEqual(['/openai/v1/models'])
|
||||
})
|
||||
|
||||
it('offers no credential when the draft names none', async () => {
|
||||
const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'm' }] }) })
|
||||
const ctx = await harness()
|
||||
|
||||
await ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url })
|
||||
|
||||
expect(server.headers[0]?.authorization).toBeUndefined()
|
||||
})
|
||||
|
||||
it('authenticates a configured route the draft cannot supply a key for', async () => {
|
||||
// What the Models page actually sends after a key is saved: the form holds
|
||||
// the redacted descriptor, so the draft names the route and the endpoint
|
||||
// and no credential at all. Interrogating unauthenticated would answer 401
|
||||
// and read as a wrong key.
|
||||
const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'm' }] }) })
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
process.env['ACME_GATEWAY_KEY'] = 'stored-key'
|
||||
touchedEnv.push('ACME_GATEWAY_KEY')
|
||||
await ctx.plugin(LlmPiAi, {
|
||||
providers: {
|
||||
'acme-gateway': {
|
||||
apiKeyEnv: 'ACME_GATEWAY_KEY',
|
||||
api: 'openai-completions',
|
||||
baseURL: server.url,
|
||||
models: [{ id: 'acme-large' }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await ctx.llm.discoverModels('llm-pi-ai', { provider: 'acme-gateway', baseURL: server.url })
|
||||
// A key typed into the form is the one being tested — possibly the
|
||||
// replacement for the stored one — so it wins.
|
||||
await ctx.llm.discoverModels('llm-pi-ai', { provider: 'acme-gateway', baseURL: server.url, apiKey: 'typed' })
|
||||
// A route no profile declares yet is the create case: nothing is stored.
|
||||
await ctx.llm.discoverModels('llm-pi-ai', { provider: 'not-declared-yet', baseURL: server.url })
|
||||
|
||||
expect(server.headers.map(headers => headers.authorization))
|
||||
.toEqual(['Bearer stored-key', 'Bearer typed', undefined])
|
||||
})
|
||||
|
||||
it('leaves a catalog route\'s credential unresolved, having never reached the network', async () => {
|
||||
// The catalog answers before any endpoint is asked, so a route whose
|
||||
// profile names a credential that is not set must still answer rather than
|
||||
// failing over a key the interrogation never needed.
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
Reflect.deleteProperty(process.env, 'ABSENT_FOR_DISCOVERY')
|
||||
await ctx.plugin(LlmPiAi, { providers: { deepseek: { apiKeyEnv: 'ABSENT_FOR_DISCOVERY' } } })
|
||||
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'deepseek' })).resolves.not.toHaveLength(0)
|
||||
})
|
||||
|
||||
it('drops unusable rows rather than failing the whole listing', async () => {
|
||||
const server = await listingServer({
|
||||
body: JSON.stringify({
|
||||
data: [
|
||||
{ id: 'good' },
|
||||
{ id: '' },
|
||||
{ name: 'no id at all' },
|
||||
null,
|
||||
{ id: 'good' },
|
||||
{ id: 'zero-capacity', context_length: 0, max_tokens: -1 },
|
||||
],
|
||||
}),
|
||||
})
|
||||
const ctx = await harness()
|
||||
|
||||
expect(await ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url }))
|
||||
.toEqual([{ id: 'good' }, { id: 'zero-capacity' }])
|
||||
})
|
||||
|
||||
it('points at the credential for a rejected one, and only then', async () => {
|
||||
const ctx = await harness()
|
||||
|
||||
for (const status of [401, 403]) {
|
||||
const refused = await listingServer({ status, body: '{"error":"nope"}' })
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: refused.url, apiKey: 'wrong' }))
|
||||
.rejects.toThrow(new RegExp(`answered ${status}; check the API key`))
|
||||
}
|
||||
|
||||
// A server fault is not a credential problem, so it must not send the user
|
||||
// off to re-check a key that is fine.
|
||||
const broken = await listingServer({ status: 500, body: '{"error":"boom"}' })
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: broken.url, apiKey: 'fine' }))
|
||||
.rejects.toThrow(/answered 500$/)
|
||||
})
|
||||
|
||||
it('reports a reply that is not a model listing', async () => {
|
||||
const server = await listingServer({ body: '{"models":[]}' })
|
||||
const ctx = await harness()
|
||||
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url }))
|
||||
.rejects.toThrow(/no "data" array; enter this provider's models by hand/)
|
||||
|
||||
const broken = await listingServer({ body: 'not json at all' })
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: broken.url }))
|
||||
.rejects.toThrow(/did not answer with JSON/)
|
||||
})
|
||||
|
||||
it('refuses an oversized reply, whether its length is declared or streamed', async () => {
|
||||
const ctx = await harness()
|
||||
// Just over the four-megabyte ceiling, as one padded model row.
|
||||
const oversized = `{"data":[{"id":"m","pad":"${'x'.repeat(4 * 1024 * 1024)}"}]}`
|
||||
|
||||
const declared = await listingServer({ body: oversized })
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: declared.url }))
|
||||
.rejects.toThrow(/answered with more than 4194304 bytes/)
|
||||
|
||||
// A streamed reply declares no length, so the ceiling has to hold on the
|
||||
// body the harness actually read.
|
||||
const streamed = await listingServer({ chunks: ['{"data":[{"id":"m","pad":"', 'x'.repeat(4 * 1024 * 1024), '"}]}'] })
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: streamed.url }))
|
||||
.rejects.toThrow(/answered with more than 4194304 bytes/)
|
||||
})
|
||||
|
||||
it('reports an unreachable endpoint instead of an empty catalog', async () => {
|
||||
const ctx = await harness()
|
||||
// Port 9 is the discard service: nothing accepts a connection there.
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: 'http://127.0.0.1:9/v1' }))
|
||||
.rejects.toMatchObject({ code: 'DISCOVERY_FAILED' })
|
||||
})
|
||||
|
||||
it.each(['anthropic-messages', 'azure-openai-responses', 'openai-codex-responses', 'google-generative-ai'])(
|
||||
'says it cannot interrogate %s rather than guessing a shape',
|
||||
async (api) => {
|
||||
// Azure authenticates with an `api-key` header and an `api-version`
|
||||
// query despite its OpenAI lineage, and Codex uses OAuth; guessing at
|
||||
// either would report an auth failure as a provider with no models.
|
||||
const ctx = await harness()
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: 'https://gateway.example/v1', api }))
|
||||
.rejects.toMatchObject({ code: 'DISCOVERY_UNSUPPORTED' })
|
||||
},
|
||||
)
|
||||
|
||||
it('reports cancellation during the body read as an abort, not a raw reason', async () => {
|
||||
const ctx = await harness()
|
||||
const controller = new AbortController()
|
||||
// Chunked, so the headers arrive and the cancellation lands mid-body.
|
||||
const slow = await listingServer({ chunks: ['{"data":[', '{"id":"a"}'], holdOpenMs: 400 })
|
||||
const probe = ctx.llm.discoverModels('llm-pi-ai', { baseURL: slow.url, signal: controller.signal })
|
||||
setTimeout(() => { controller.abort('test cancellation') }, 40)
|
||||
|
||||
await expect(probe).rejects.toMatchObject({ code: 'ABORTED' })
|
||||
})
|
||||
|
||||
it('honors caller cancellation', async () => {
|
||||
const ctx = await harness()
|
||||
const aborted = AbortSignal.abort('test cancellation')
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', {
|
||||
baseURL: 'http://127.0.0.1:9/v1',
|
||||
signal: aborted,
|
||||
})).rejects.toMatchObject({ code: 'ABORTED' })
|
||||
})
|
||||
|
||||
it('is offered for the namespace, and refuses one it does not serve', async () => {
|
||||
const ctx = await harness()
|
||||
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'openai' })).resolves.not.toHaveLength(0)
|
||||
await expect(ctx.llm.discoverModels('llm-deepseek', { baseURL: 'https://api.deepseek.com' }))
|
||||
.rejects.toMatchObject({ code: 'NO_DISCOVERY' })
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: '' }))
|
||||
.rejects.toMatchObject({ code: 'INVALID_DISCOVERY' })
|
||||
})
|
||||
|
||||
it('withdraws the offer when the plugin unloads', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const fiber = await ctx.plugin(LlmPiAi, {})
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'openai' })).resolves.not.toHaveLength(0)
|
||||
|
||||
await fiber.dispose()
|
||||
|
||||
await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'openai' }))
|
||||
.rejects.toMatchObject({ code: 'NO_DISCOVERY' })
|
||||
})
|
||||
})
|
||||
@@ -155,13 +155,16 @@ describe('request-level dynamic profiles', () => {
|
||||
expect(ctx.llm.listProviders().map(provider => provider.id)).toEqual(['openai'])
|
||||
})
|
||||
|
||||
it('keeps the last good profiles when a settings snapshot names an unknown provider', async () => {
|
||||
it('refuses a settings write this adapter could not serve, leaving its routes alone', async () => {
|
||||
const dir = await home()
|
||||
const ctx = await boot(dir, { providers: { openai: {} } })
|
||||
|
||||
// Schema-valid but catalog-invalid: the resolver rejects it and the
|
||||
// last good route set keeps serving.
|
||||
await ctx.settings.update(NS, { providers: { 'not-a-real-provider': {} } })
|
||||
// Shape-valid but unserviceable: a route the catalog does not ship and
|
||||
// that lists no models of its own. The section schema resolves the whole
|
||||
// profile set, so this is refused where it is written rather than stored
|
||||
// and then quietly disabling every route in the namespace.
|
||||
await expect(ctx.settings.update(NS, { providers: { 'not-a-real-provider': {} } }))
|
||||
.rejects.toThrow(/resolves no models/)
|
||||
expect(ctx.llm.listProviders().map(provider => provider.id)).toEqual(['openai'])
|
||||
})
|
||||
|
||||
|
||||
@@ -1,41 +1,73 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
|
||||
const streamSimple = vi.hoisted(() => vi.fn())
|
||||
|
||||
// The 0.81 SDK moved `streamSimple` to the compat entry; the adapter imports it
|
||||
// from there, so the mock must target the same specifier.
|
||||
vi.mock('@earendil-works/pi-ai/compat', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@earendil-works/pi-ai/compat')>()
|
||||
return { ...actual, streamSimple }
|
||||
})
|
||||
// A hand-declared route is built by `createProvider` over the protocol table in
|
||||
// `src/provider.ts`, so the table's lazy api module is the SDK boundary this
|
||||
// test can observe. A catalog route dispatches through pi-ai's own provider and
|
||||
// would not see this mock.
|
||||
vi.mock('@earendil-works/pi-ai/api/openai-completions.lazy', () => ({
|
||||
openAICompletionsApi: () => ({ stream: streamSimple, streamSimple }),
|
||||
}))
|
||||
|
||||
import { PiAiAdapter } from '../src/adapter.ts'
|
||||
import { resolveProfiles } from '../src/config.ts'
|
||||
|
||||
afterEach(() => { streamSimple.mockReset() })
|
||||
|
||||
/** A hand-declared OpenAI-compatible route with one fully described model. */
|
||||
function gatewayAdapter(): PiAiAdapter {
|
||||
return new PiAiAdapter({
|
||||
profiles: () => resolveProfiles({
|
||||
'local-gateway': {
|
||||
api: 'openai-completions',
|
||||
baseURL: 'http://127.0.0.1:9/v1',
|
||||
models: [{ id: 'local-model', contextWindow: 8192, maxTokens: 1024 }],
|
||||
},
|
||||
}),
|
||||
resolveApiKey: () => Promise.resolve('test-key'),
|
||||
})
|
||||
}
|
||||
|
||||
async function drain(adapter: PiAiAdapter): Promise<StreamChunk[]> {
|
||||
const chunks: StreamChunk[] = []
|
||||
for await (const chunk of adapter.stream({
|
||||
provider: 'local-gateway',
|
||||
model: 'local-model',
|
||||
messages: [],
|
||||
})) chunks.push(chunk)
|
||||
return chunks
|
||||
}
|
||||
|
||||
describe('pi-ai SDK retry boundary', () => {
|
||||
it('pins one SDK attempt even when the installed provider currently defaults to zero retries', async () => {
|
||||
const failure = new Error('mock SDK boundary')
|
||||
streamSimple.mockReturnValue({
|
||||
async * [Symbol.asyncIterator](): AsyncGenerator<never> {
|
||||
throw failure
|
||||
},
|
||||
})
|
||||
const adapter = new PiAiAdapter({
|
||||
profiles: () => resolveProfiles({ openai: {} }),
|
||||
resolveApiKey: () => Promise.resolve('test-key'),
|
||||
})
|
||||
const drain = async (): Promise<void> => {
|
||||
for await (const _chunk of adapter.stream({
|
||||
provider: 'openai',
|
||||
model: 'gpt-4.1',
|
||||
messages: [],
|
||||
})) { /* drain */ }
|
||||
}
|
||||
streamSimple.mockImplementation(() => { throw new Error('mock SDK boundary') })
|
||||
|
||||
const chunks = await drain(gatewayAdapter())
|
||||
|
||||
await expect(drain()).rejects.toBe(failure)
|
||||
expect(streamSimple).toHaveBeenCalledOnce()
|
||||
expect(streamSimple.mock.calls[0]?.[2]).toMatchObject({ maxRetries: 0 })
|
||||
expect(streamSimple.mock.calls[0]?.[2]).toMatchObject({ maxRetries: 0, apiKey: 'test-key' })
|
||||
// pi-ai reports a setup failure as a terminal in-stream error rather than
|
||||
// throwing, which the converter turns into the harness error finish.
|
||||
expect(chunks.at(-1)).toMatchObject({
|
||||
type: 'finish',
|
||||
reason: { kind: 'error', failure: { message: 'mock SDK boundary' } },
|
||||
})
|
||||
})
|
||||
|
||||
it('dispatches a hand-declared route to the endpoint and model its configuration describes', async () => {
|
||||
streamSimple.mockImplementation(() => { throw new Error('mock SDK boundary') })
|
||||
|
||||
await drain(gatewayAdapter())
|
||||
|
||||
expect(streamSimple.mock.calls[0]?.[0]).toMatchObject({
|
||||
id: 'local-model',
|
||||
provider: 'local-gateway',
|
||||
api: 'openai-completions',
|
||||
baseUrl: 'http://127.0.0.1:9/v1',
|
||||
contextWindow: 8192,
|
||||
maxTokens: 1024,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user