fix(llm-pi-ai): let a model declare the request modalities it accepts

A model the installed pi-ai catalog does not describe was reported as
text-only with no way to say otherwise, so a vision model added through
the custom-provider form was refused at every image admission point.
The justification in the source described the DeepSeek chat-completions
serializer, which does reject image blocks; the pi-ai request converter
and every wire protocol it speaks carry images.

Modalities now resolve entry `input` -> installed catalog entry -> route
`defaultInput`, the chain the two capacity fallbacks already use, so the
route value is a fallback and never narrows a catalog model. Its default
stays `[text]`: nothing can interrogate a gateway for its modalities, and
over-claiming admits an image the provider rejects mid-turn, after prompt
admission has already committed the message.
This commit is contained in:
Yichen Jiang
2026-08-12 15:11:46 +08:00
parent e9dfb30cdf
commit 94289d057e
15 changed files with 443 additions and 19 deletions
@@ -186,6 +186,75 @@ describe('hand-declared providers', () => {
expect(resolved.get('acme-gateway')?.configuredMaxTokens.get('sized')).toBe(512)
})
it('takes a model’s declared modalities, then the catalog’s, then the route’s', () => {
const vision = getBuiltinModels('anthropic').find(model => model.input.includes('image'))
if (vision === undefined) throw new Error('the installed catalog ships no anthropic vision model')
const resolved = resolveProfiles({
'acme-gateway': {
api: 'openai-completions',
baseURL: 'https://acme.test',
// One route, two modality sets: the entry field is what says so.
models: [{ id: 'bare' }, { id: 'seeing', input: ['text', 'image'] }, { id: 'deaf', input: ['text'] }],
},
'seeing-gateway': {
api: 'openai-completions',
baseURL: 'https://seeing.test',
// A gateway whose undescribed models all take images says so once
// rather than on every entry; an entry still outranks it.
defaultInput: ['text', 'image'],
models: [{ id: 'bare' }, { id: 'deaf', input: ['text'] }],
},
// The route value is a fallback, never an override: a catalog model
// keeps what the catalog records even under a narrower route default,
// exactly as it keeps its own contextWindow.
'anthropic': { defaultInput: ['text'] },
})
const inputOf = (route: string, id: string): readonly string[] | undefined =>
resolved.get(route)?.piProvider.getModels().find(model => model.id === id)?.input
// Nothing can interrogate the endpoint, so an undeclared model on a route
// that names no default claims only the modality every supported protocol
// carries. Claiming images instead would admit one the provider then
// rejects mid-turn, after the durable message is already in the log.
expect(inputOf('acme-gateway', 'bare')).toEqual(['text'])
expect(inputOf('acme-gateway', 'seeing')).toEqual(['text', 'image'])
expect(inputOf('acme-gateway', 'deaf')).toEqual(['text'])
expect(inputOf('seeing-gateway', 'bare')).toEqual(['text', 'image'])
expect(inputOf('seeing-gateway', 'deaf')).toEqual(['text'])
expect(inputOf('anthropic', vision.id)).toEqual(vision.input)
})
it('reads an entry’s empty modality list as no answer, and the route’s as unserviceable', () => {
// Absent and empty are the same request on an entry, exactly as they are
// for the route's `models` list — which matters because the config schema
// materializes `[]` for an absent array, so an entry naming a catalog
// model without declaring modalities must keep the catalog's rather than
// describe a model that accepts nothing.
const [catalogModel] = getBuiltinModels('deepseek')
if (catalogModel === undefined) throw new Error('the installed catalog ships no deepseek model')
const resolved = resolveProfiles({
'deepseek': { baseURL: 'https://catalog.test', models: [{ id: catalogModel.id, input: [] }] },
'acme-gateway': {
api: 'openai-completions',
baseURL: 'https://acme.test',
models: [{ id: 'bare', input: [] }],
},
})
expect(resolved.get('acme-gateway')?.piProvider.getModels()[0]?.input).toEqual(['text'])
expect(resolved.get('deepseek')?.piProvider.getModels()[0]?.input).toEqual(catalogModel.input)
// Nothing sits below the route value, so its empty list states no answer
// anything could take, and is refused where it is written.
expect(() => resolveProfiles({
'acme-gateway': {
api: 'openai-completions',
baseURL: 'https://acme.test',
defaultInput: [],
models: [{ id: 'bare' }],
},
})).toThrow(/defaultInput must name at least one modality/)
})
it('rejects a model the route cannot identify', () => {
const declare = (model: LlmPiAi.PiAiModelProfile): (() => unknown) =>
() => resolveProfiles({ 'acme-gateway': { api: 'openai-completions', baseURL: 'https://acme.test', models: [model] } })