Merge remote-tracking branch 'refs/remotes/origin/worktree/web-multimodal-image-input' into worktree/pr555-simplify
# Conflicts: # .agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml # .agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md # .agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md # apps/cli/README.i18n.yaml # apps/cli/README.md # apps/cli/README.zh.md # apps/cli/src/app-cli-entry.ts # packages/client/connection/src/client/fixture.ts # packages/client/ui-conversation/src/client/service.ts # packages/host/apiproxy/src/api-proxy.ts # packages/host/apiproxy/src/api/host.schema.ts # packages/host/apiproxy/src/api/host.ts # packages/host/apiproxy/tests/fetch-carrier.spec.ts
This commit is contained in:
@@ -196,9 +196,9 @@ export function apply(ctx: Context): void {
|
||||
const shell = inputHub.shell(sessionId)
|
||||
return {
|
||||
keyboard: shell,
|
||||
addImages: (files, current) => {
|
||||
addImages: (files) => {
|
||||
try {
|
||||
const images = conversation.createDraftImages(files, current)
|
||||
const images = conversation.createDraftImages(files)
|
||||
shell.addImages(images.map(image => image.id))
|
||||
return null
|
||||
} catch (error: unknown) {
|
||||
|
||||
@@ -273,7 +273,7 @@ export interface ComposerBarInjected {
|
||||
/** The InputBar-exclusive keyboard/DOM command face (decision 20 private plane). */
|
||||
keyboard: ComposerKeyboard
|
||||
/** Create browser previews and append their ids to the session input state. */
|
||||
addImages: (files: readonly File[], current: readonly ComposerAttachment[]) => string | null
|
||||
addImages: (files: readonly File[]) => string | null
|
||||
/** Release one browser preview and remove its id from the session input state. */
|
||||
removeImage: (id: string) => void
|
||||
/** Resolve ordered input-state ids to browser-owned draft attachments. */
|
||||
|
||||
@@ -123,7 +123,6 @@ export class ConversationService extends Service implements IConversation {
|
||||
mode: 'queue' | 'steer',
|
||||
images: readonly File[],
|
||||
): Promise<void> {
|
||||
this.validateImages(images, [])
|
||||
const uploaded = await this.serializeImages(images)
|
||||
const content = [...uploaded, ...(text === '' ? [] : [{ type: 'text' as const, text }])]
|
||||
const result = await session.prompt(content, mode)
|
||||
@@ -133,14 +132,10 @@ export class ConversationService extends Service implements IConversation {
|
||||
/**
|
||||
* Create runtime-only draft attachments and their object URLs.
|
||||
* @param files - browser-owned image files.
|
||||
* @param current - images already present in the same composer.
|
||||
* @returns ordered attachment descriptors whose ids may enter the input state.
|
||||
*/
|
||||
createDraftImages(
|
||||
files: readonly File[],
|
||||
current: readonly ComposerAttachment[] = [],
|
||||
): readonly ComposerAttachment[] {
|
||||
this.validateImages(files, current)
|
||||
createDraftImages(files: readonly File[]): readonly ComposerAttachment[] {
|
||||
for (const file of files) imageMediaType(file.type)
|
||||
return files.map((file) => {
|
||||
const attachment = browserDraftAttachment(file)
|
||||
this.draftAttachments.set(attachment.id, attachment)
|
||||
@@ -276,36 +271,6 @@ export class ConversationService extends Service implements IConversation {
|
||||
return sessions
|
||||
}
|
||||
|
||||
/** Apply host-advertised fast-path checks before any object URL or base64 allocation. */
|
||||
private validateImages(
|
||||
files: readonly File[],
|
||||
current: readonly ComposerAttachment[],
|
||||
): void {
|
||||
if (files.length === 0 && current.length === 0) return
|
||||
// Model capability is checked only by the host against the session's
|
||||
// current target; the client owns deployment upload limits.
|
||||
const description = this.requireSessions().hostDescription()
|
||||
const limits = description?.imageLimits
|
||||
const all = [...current.map(attachment => attachment.file), ...files]
|
||||
if (limits !== undefined && all.length > limits.maxImagesPerMessage) {
|
||||
throw new Error(`每条消息最多添加 ${limits.maxImagesPerMessage} 张图片`)
|
||||
}
|
||||
let totalBytes = 0
|
||||
for (const file of all) {
|
||||
const mediaType = imageMediaType(file.type)
|
||||
if (limits !== undefined && !limits.mediaTypes.includes(mediaType)) {
|
||||
throw new Error(`当前部署不支持 ${mediaType} 图片`)
|
||||
}
|
||||
if (limits !== undefined && file.size > limits.maxImageBytes) {
|
||||
throw new Error(`图片 ${file.name || '未命名图片'} 超过单张大小限制`)
|
||||
}
|
||||
totalBytes += file.size
|
||||
}
|
||||
if (limits !== undefined && totalBytes > limits.maxMessageImageBytes) {
|
||||
throw new Error('图片总大小超过单条消息限制')
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert browser files to the prompt wire's canonical base64 image parts. */
|
||||
private serializeImages(images: readonly File[]): Promise<Parameters<SessionFace['prompt']>[0]> {
|
||||
return Promise.all(images.map(async file => ({
|
||||
|
||||
@@ -234,7 +234,7 @@ export function InputBar({
|
||||
.map(item => item.getAsFile())
|
||||
.filter((file): file is File => file !== null)
|
||||
if (files.length > 0) {
|
||||
setDropError(addImages(files, attachments))
|
||||
setDropError(addImages(files))
|
||||
}
|
||||
const text = e.clipboardData.getData('text/plain')
|
||||
if (text === '') {
|
||||
@@ -290,7 +290,7 @@ export function InputBar({
|
||||
if (locked || machineBusy) return
|
||||
const dropped = [...event.dataTransfer.files]
|
||||
if (dropped.length === 0) return
|
||||
setDropError(addImages(dropped, attachments))
|
||||
setDropError(addImages(dropped))
|
||||
}
|
||||
|
||||
const closePreview = useCallback(() => { setPreview(null) }, [])
|
||||
|
||||
@@ -49,7 +49,7 @@ interface BenchOptions {
|
||||
leftItems?: React.ReactNode
|
||||
rightItems?: React.ReactNode
|
||||
attachments?: readonly ComposerAttachment[]
|
||||
addImages?: (files: readonly File[], current: readonly ComposerAttachment[]) => string | null
|
||||
addImages?: (files: readonly File[]) => string | null
|
||||
}
|
||||
|
||||
/** Real machine behind the bar entry: sink spy, no slash pipeline (plain text goes straight to the sink). */
|
||||
@@ -464,7 +464,7 @@ describe('image draft rail', () => {
|
||||
getData: () => '同时粘贴的文字',
|
||||
},
|
||||
})
|
||||
expect(addImages).toHaveBeenCalledWith([image], [])
|
||||
expect(addImages).toHaveBeenCalledWith([image])
|
||||
expect(shell.snapshot.draft).toBe('同时粘贴的文字')
|
||||
|
||||
const video = new File([Uint8Array.of(1)], 'clip.mp4', { type: 'video/mp4' })
|
||||
@@ -494,7 +494,7 @@ describe('image draft rail', () => {
|
||||
expect(dataTransfer.dropEffect).toBe('copy')
|
||||
expect(fireEvent.drop(card, { dataTransfer })).toBe(false)
|
||||
expect(view.queryByRole('status')).toBeNull()
|
||||
expect(addImages).toHaveBeenCalledWith([image], [])
|
||||
expect(addImages).toHaveBeenCalledWith([image])
|
||||
})
|
||||
|
||||
it('ignores unsupported dropped files and refuses drops while locked', () => {
|
||||
@@ -507,7 +507,7 @@ describe('image draft rail', () => {
|
||||
dataTransfer: { types: ['Files'], files: [documentFile], dropEffect: 'none' },
|
||||
})
|
||||
expect(view.getByText(/不支持的图片格式/)).toBeTruthy()
|
||||
expect(addImages).toHaveBeenCalledWith([documentFile], [])
|
||||
expect(addImages).toHaveBeenCalledWith([documentFile])
|
||||
|
||||
const image = new File([Uint8Array.of(1)], 'locked.png', { type: 'image/png' })
|
||||
const locked = bench({ disabled: true, addImages })
|
||||
|
||||
@@ -49,47 +49,6 @@ describe('ConversationService', () => {
|
||||
await b.runtime.dispose()
|
||||
})
|
||||
|
||||
it('accepts ordered batches and preflights their advertised count and aggregate limits', async () => {
|
||||
const b = await bench()
|
||||
const described = vi.spyOn(b.runtime.sessions, 'hostDescription').mockReturnValue({
|
||||
version: 'test',
|
||||
cwd: '/tmp',
|
||||
imageLimits: {
|
||||
maxImageBytes: 3,
|
||||
maxImagesPerMessage: 2,
|
||||
maxMessageImageBytes: 3,
|
||||
maxImagePixels: 4,
|
||||
mediaTypes: ['image/png'],
|
||||
},
|
||||
attachedSessions: 1,
|
||||
})
|
||||
const created = vi.spyOn(URL, 'createObjectURL')
|
||||
.mockReturnValueOnce('blob:first')
|
||||
.mockReturnValueOnce('blob:second')
|
||||
const revoked = vi.spyOn(URL, 'revokeObjectURL').mockReturnValue(undefined)
|
||||
try {
|
||||
const attachments = b.root.createDraftImages([
|
||||
new File([Uint8Array.of(1)], 'first.png', { type: 'image/png' }),
|
||||
new File([Uint8Array.of(2)], 'second.png', { type: 'image/png' }),
|
||||
])
|
||||
expect(attachments.map(attachment => attachment.file.name)).toEqual(['first.png', 'second.png'])
|
||||
expect(() => b.root.createDraftImages([
|
||||
new File([Uint8Array.of(3)], 'third.png', { type: 'image/png' }),
|
||||
], attachments)).toThrow('每条消息最多添加 2 张图片')
|
||||
const first = attachments[0]
|
||||
if (first === undefined) throw new Error('first draft attachment missing')
|
||||
expect(() => b.root.createDraftImages([
|
||||
new File([Uint8Array.of(3, 4, 5)], 'large.png', { type: 'image/png' }),
|
||||
], [first])).toThrow('图片总大小超过单条消息限制')
|
||||
expect(created).toHaveBeenCalledTimes(2)
|
||||
} finally {
|
||||
await b.runtime.dispose()
|
||||
described.mockRestore()
|
||||
created.mockRestore()
|
||||
revoked.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('releases draft images when the session scope is disposed', async () => {
|
||||
const b = await bench()
|
||||
const created = vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:draft-1')
|
||||
@@ -110,6 +69,31 @@ describe('ConversationService', () => {
|
||||
await b.runtime.dispose()
|
||||
})
|
||||
|
||||
it('checks media type before preview allocation and leaves deployment limits to the host', async () => {
|
||||
const b = await bench()
|
||||
const created = vi.spyOn(URL, 'createObjectURL').mockImplementation(file => `blob:${(file as File).name}`)
|
||||
try {
|
||||
const files = Array.from(
|
||||
{ length: 11 },
|
||||
(_, index) => new File([Uint8Array.of(index)], `${index}.png`, { type: 'image/png' }),
|
||||
)
|
||||
expect(b.root.createDraftImages(files)).toHaveLength(11)
|
||||
expect(created).toHaveBeenCalledTimes(11)
|
||||
|
||||
const beforeRejectedBatch = created.mock.calls.length
|
||||
expect(() => {
|
||||
b.root.createDraftImages([
|
||||
new File([Uint8Array.of(1)], 'valid.png', { type: 'image/png' }),
|
||||
new File([Uint8Array.of(2)], 'invalid.svg', { type: 'image/svg+xml' }),
|
||||
])
|
||||
}).toThrow('不支持的图片格式:image/svg+xml')
|
||||
expect(created).toHaveBeenCalledTimes(beforeRejectedBatch)
|
||||
} finally {
|
||||
created.mockRestore()
|
||||
}
|
||||
await b.runtime.dispose()
|
||||
})
|
||||
|
||||
it('releases in-flight send images when the scope dies before the failure lands', async () => {
|
||||
const b = await bench()
|
||||
const created = vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:inflight-1')
|
||||
|
||||
Reference in New Issue
Block a user