Add web multimodal image attachments

2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions
@@ -132,25 +132,25 @@ describe('conversation slot inject surface', () => {
    const { instance, injected } = b.conversationSurface(ROOT)
    // Whitespace-only: no send, and the (whitespace) draft is not cleared.
    instance.actions.setDraft('   ')
-    injected.send('   ', 'queue')
+    injected.send('   ', [], 'queue')
    expect(b.sessionFake.prompt).not.toHaveBeenCalled()
    expect(instance.store.getSnapshot().draft).toBe('   ')
    // Success: cleared and stays cleared.
    instance.actions.setDraft('hello')
-    injected.send('hello', 'queue')
+    injected.send('hello', [], 'queue')
    expect(instance.store.getSnapshot().draft).toBe('')
    await Promise.resolve()
    expect(b.sessionFake.prompt).toHaveBeenCalledWith([{ type: 'text', text: 'hello' }], 'queue')
    // Failure: restored (draft still empty when the rejection lands).
    b.sessionFake.prompt.mockResolvedValueOnce({ ok: false, error: { code: 'agent-busy', message: 'b' } })
    instance.actions.setDraft('retry me')
-    injected.send('retry me', 'queue')
+    injected.send('retry me', [], 'queue')
    await vi.waitFor(() => {
      expect(instance.store.getSnapshot().draft).toBe('retry me')
    })
    // Failure landing after new typing: no clobber (restoreDraft fills empty only).
    b.sessionFake.prompt.mockResolvedValueOnce({ ok: false, error: { code: 'agent-busy', message: 'b' } })
-    injected.send('retry me', 'queue')
+    injected.send('retry me', [], 'queue')
    instance.actions.setDraft('typed during flight')
    await new Promise(r => setTimeout(r, 0))
    expect(instance.store.getSnapshot().draft).toBe('typed during flight')
@@ -15,9 +15,9 @@ beforeEach(() => {
 })

 describe('createChatStore', () => {
-  it('init shape: empty selection/draft/view', () => {
+  it('init shape: empty selection/draft/images/view', () => {
    const store = createChatStore().create()
-    expect(store.store.getSnapshot()).toEqual({ selection: null, draft: '', view: null })
+    expect(store.store.getSnapshot()).toEqual({ selection: null, draft: '', imageIds: [], view: null })
  })

  it('actions cover the declared write set', () => {
@@ -30,6 +30,11 @@ describe('createChatStore', () => {

    store.actions.setDraft('hello')
    expect(store.store.getSnapshot().draft).toBe('hello')
+    store.actions.addImages(['a', 'b'])
+    store.actions.removeImage('a')
+    expect(store.store.getSnapshot().imageIds).toEqual(['b'])
+    store.actions.pruneImages([])
+    expect(store.store.getSnapshot().imageIds).toEqual([])
    store.actions.clearDraft()
    expect(store.store.getSnapshot().draft).toBe('')

@@ -40,12 +45,15 @@ describe('createChatStore', () => {
  it('restoreDraft only fills an empty draft (optimistic-send rollback contract)', () => {
    const store = createChatStore().create()
    // Rollback path: draft was cleared by send, nothing typed since.
-    store.actions.restoreDraft('failed text')
+    store.actions.restoreDraft('failed text', ['old-image'])
    expect(store.store.getSnapshot().draft).toBe('failed text')
+    expect(store.store.getSnapshot().imageIds).toEqual(['old-image'])
    // The user typed something new before the failure landed: keep theirs.
    store.actions.setDraft('newer input')
-    store.actions.restoreDraft('stale text')
+    store.actions.addImages(['new-image'])
+    store.actions.restoreDraft('stale text', ['old-image'])
    expect(store.store.getSnapshot().draft).toBe('newer input')
+    expect(store.store.getSnapshot().imageIds).toEqual(['old-image', 'new-image'])
  })

  it('persists per scope key and rehydrates a fresh instance', () => {
@@ -129,3 +129,89 @@ describe('error strip and variants', () => {
    expect(view.container.querySelector('[class*="hero"]')).not.toBeNull()
  })
 })
+
+describe('image draft rail', () => {
+  it('collects supported clipboard images and leaves non-image clipboard data to the browser', () => {
+    const onAddImages = vi.fn()
+    const { textarea } = setup({ draft: '', onAddImages })
+    const image = new File([Uint8Array.of(1, 2, 3)], 'pixel.png', { type: 'image/png' })
+    const prevented = fireEvent.paste(textarea, {
+      clipboardData: {
+        items: [
+          { kind: 'string', type: 'text/plain', getAsFile: () => null },
+          { kind: 'file', type: 'image/png', getAsFile: () => image },
+        ],
+      },
+    })
+    expect(prevented).toBe(false)
+    expect(onAddImages).toHaveBeenCalledWith([image])
+
+    fireEvent.paste(textarea, {
+      clipboardData: { items: [{ kind: 'file', type: 'video/mp4', getAsFile: () => image }] },
+    })
+    expect(onAddImages).toHaveBeenCalledTimes(1)
+  })
+
+  it('accepts supported image drops, highlights the target, and prevents browser navigation', () => {
+    const onAddImages = vi.fn()
+    const { view } = setup({ draft: '', onAddImages })
+    const card = view.container.querySelector('[class*="card"]')!
+    const image = new File([Uint8Array.of(1, 2, 3)], 'dropped.png', { type: 'image/png' })
+    const dataTransfer = {
+      types: ['Files'],
+      files: [image],
+      dropEffect: 'none',
+    }
+    expect(fireEvent.dragEnter(card, { dataTransfer })).toBe(false)
+    expect(view.getByRole('status').textContent).toContain('松开以添加图片')
+    expect(fireEvent.dragOver(card, { dataTransfer })).toBe(false)
+    expect(dataTransfer.dropEffect).toBe('copy')
+    expect(fireEvent.drop(card, { dataTransfer })).toBe(false)
+    expect(view.queryByRole('status')).toBeNull()
+    expect(onAddImages).toHaveBeenCalledWith([image])
+  })
+
+  it('ignores unsupported dropped files and refuses drops while locked', () => {
+    const onAddImages = vi.fn()
+    const { view } = setup({ draft: '', onAddImages })
+    const card = view.container.querySelector('[class*="card"]')!
+    const documentFile = new File(['hello'], 'notes.txt', { type: 'text/plain' })
+    fireEvent.drop(card, {
+      dataTransfer: { types: ['Files'], files: [documentFile], dropEffect: 'none' },
+    })
+    expect(view.getByText(/暂仅支持 PNG/)).toBeTruthy()
+    expect(onAddImages).not.toHaveBeenCalled()
+
+    const image = new File([Uint8Array.of(1)], 'locked.png', { type: 'image/png' })
+    const locked = setup({ draft: '', disabled: true, onAddImages })
+    const lockedCard = locked.view.container.querySelector('[class*="card"]')!
+    const dataTransfer = { types: ['Files'], files: [image], dropEffect: 'copy' }
+    fireEvent.dragEnter(lockedCard, { dataTransfer })
+    expect(locked.view.queryByRole('status')).toBeNull()
+    fireEvent.dragOver(lockedCard, { dataTransfer })
+    expect(dataTransfer.dropEffect).toBe('none')
+    fireEvent.drop(lockedCard, { dataTransfer })
+    expect(onAddImages).not.toHaveBeenCalled()
+  })
+
+  it('allows image-only send, removes a thumbnail, and opens original preview on double-click', () => {
+    const file = new File([Uint8Array.of(1)], 'pixel.png', { type: 'image/png' })
+    const attachment = { id: 'draft-1', file, previewUrl: 'blob:draft-1' }
+    const onRemoveAttachment = vi.fn()
+    const { view, textarea, props } = setup({
+      draft: '', attachments: [attachment], onRemoveAttachment,
+    })
+    const send = view.getByRole('button', { name: '发送' }) as HTMLButtonElement
+    expect(send.disabled).toBe(false)
+    fireEvent.keyDown(textarea, { key: 'Enter' })
+    expect(props.onSend).toHaveBeenCalledWith('queue')
+
+    fireEvent.click(view.getByRole('button', { name: '移除图片 pixel.png' }))
+    expect(onRemoveAttachment).toHaveBeenCalledWith('draft-1')
+    fireEvent.doubleClick(view.getByTitle('双击查看原图'))
+    expect(view.getByRole('dialog', { name: '原图预览' })).toBeTruthy()
+    expect(view.getAllByAltText('pixel.png').every(node => (node as HTMLImageElement).src.includes('blob:draft-1'))).toBe(true)
+    fireEvent.keyDown(window, { key: 'Escape' })
+    expect(view.queryByRole('dialog', { name: '原图预览' })).toBeNull()
+  })
+})
@@ -0,0 +1,44 @@
+// @vitest-environment jsdom
+
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { cleanup, fireEvent, render, waitFor } from '@testing-library/react'
+import { AttachmentId } from '@deepseek-ai/dsh-attachment'
+import { MessageImage } from '../src/client/chat/MessageImage.tsx'
+
+afterEach(cleanup)
+
+const attachment = {
+  attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
+  mediaType: 'image/png' as const,
+  bytes: 68,
+  width: 640,
+  height: 320,
+  name: 'history.png',
+}
+
+describe('MessageImage', () => {
+  it('loads a session-authorized URL, bounds the thumbnail, and double-clicks into the original', async () => {
+    const load = vi.fn().mockResolvedValue('blob:history')
+    const view = render(<MessageImage attachment={attachment} load={load} />)
+    const frame = view.getByRole('button', { name: 'history.png，双击查看原图' })
+    expect(frame.getAttribute('style')).toContain('width: 240px')
+    expect(frame.getAttribute('style')).toContain('height: 120px')
+    await waitFor(() => { expect(view.getByAltText('history.png')).toBeTruthy() })
+    expect(load).toHaveBeenCalledWith(attachment)
+    fireEvent.doubleClick(frame)
+    expect(view.getByRole('dialog', { name: '原图预览' })).toBeTruthy()
+    fireEvent.click(view.getByRole('button', { name: '关闭原图预览' }))
+    expect(view.queryByRole('dialog', { name: '原图预览' })).toBeNull()
+  })
+
+  it('surfaces a retry control when durable bytes cannot be read', async () => {
+    const load = vi.fn()
+      .mockRejectedValueOnce(new Error('offline'))
+      .mockResolvedValueOnce('blob:retry')
+    const view = render(<MessageImage attachment={attachment} load={load} />)
+    const retry = await view.findByRole('button', { name: '图片加载失败，点击重试' })
+    fireEvent.click(retry)
+    await waitFor(() => { expect(view.getByAltText('history.png')).toBeTruthy() })
+    expect(load).toHaveBeenCalledTimes(2)
+  })
+})
@@ -175,6 +175,6 @@ describe('selection survives on the store seat', () => {
    await flush()
    const reborn = storeFor(b, 'conversation', sid('s1'))
    expect(reborn).not.toBe(doomed)
-    expect(reborn.store.getSnapshot()).toEqual({ selection: null, draft: '', view: null })
+    expect(reborn.store.getSnapshot()).toEqual({ selection: null, draft: '', imageIds: [], view: null })
  })
 })
@@ -93,6 +93,29 @@ describe('send / cancel', () => {
    await expect(s.send('x', 'queue')).rejects.toThrow(/send failed: agent-busy: busy/)
  })

+  it('uploads temporary browser files as base64 image parts at the send boundary', async () => {
+    const b = await bench()
+    const file = new File([Uint8Array.of(1, 2, 3)], 'pixel.png', { type: 'image/png' })
+    Object.defineProperty(file, 'arrayBuffer', {
+      value: () => Promise.resolve(Uint8Array.of(1, 2, 3).buffer),
+    })
+    await b.scopedSvc(sid('s1')).send('describe', 'queue', [file])
+    expect(b.sessionDoubles.get(sid('s1'))!.prompt).toHaveBeenCalledWith([
+      { type: 'image', mediaType: 'image/png', data: 'AQID', name: 'pixel.png' },
+      { type: 'text', text: 'describe' },
+    ], 'queue')
+  })
+
+  it('rejects unsupported browser media before prompting the session', async () => {
+    const b = await bench()
+    const file = new File([Uint8Array.of(1)], 'clip.mp4', { type: 'video/mp4' })
+    Object.defineProperty(file, 'arrayBuffer', {
+      value: () => Promise.resolve(Uint8Array.of(1).buffer),
+    })
+    await expect(b.scopedSvc(sid('s1')).send('', 'queue', [file])).rejects.toThrow(/不支持的图片格式/)
+    expect(b.sessionDoubles.get(sid('s1'))?.prompt).not.toHaveBeenCalled()
+  })
+
  it('cancel resolves on ok and throws the folded business error', async () => {
    const b = await bench()
    const s = b.scopedSvc(sid('s1'))
@@ -71,6 +71,9 @@ describe('ConversationRoot branches', () => {
        useStore={hookOf(chat)}
        actions={chat.actions}
        views={{ list: () => [chatEntry], subscribe: () => () => {}, version: () => 1 }}
+        addImages={vi.fn()}
+        removeImage={vi.fn()}
+        draftImages={() => []}
        send={vi.fn()}
        stop={vi.fn()}
        openDetails={vi.fn()}
@@ -129,6 +132,9 @@ describe('ConversationRoot branches', () => {
        useStore={hookOf(chat)}
        actions={chat.actions}
        views={{ list: () => [chatEntry], subscribe: () => () => {}, version: () => 1 }}
+        addImages={vi.fn()}
+        removeImage={vi.fn()}
+        draftImages={() => []}
        send={vi.fn()}
        stop={vi.fn()}
        openDetails={vi.fn()}
@@ -121,6 +121,9 @@ describe('ConversationRoot', () => {
          subscribe: () => () => {},
          version: () => 1,
        }}
+        addImages={vi.fn()}
+        removeImage={vi.fn()}
+        draftImages={() => []}
        send={send}
        stop={stop}
        openDetails={openDetails}
@@ -183,7 +186,7 @@ describe('ConversationRoot', () => {
    // Typing goes through actions.setDraft into the shared store.
    expect(chat.store.getSnapshot().draft).toBe('hi')
    fireEvent.keyDown(box, { key: 'Enter' })
-    expect(send).toHaveBeenCalledWith('hi', 'queue')
+    expect(send).toHaveBeenCalledWith('hi', [], 'queue')
  })
 })