fix(gui): harden multimodal image attachments

2026-07-23 19:38:37 +08:00
parent eea595fcb4
commit 580e05b794
61 changed files with 1700 additions and 214 deletions
@@ -94,15 +94,21 @@ export function apply(ctx: Context): void {
          subscribe: fn => conversation.subscribeViews(fn),
          version: () => conversation.viewsVersion(),
        },
-        addImages: (files) => {
-          const images = conversation.createDraftImages(files)
-          actions.addImages(images.map(image => image.id))
+        addImages: (files, current) => {
+          try {
+            const images = conversation.createDraftImages(files, current)
+            actions.addImages(images.map(image => image.id))
+            return null
+          } catch (error: unknown) {
+            return error instanceof Error ? error.message : String(error)
+          }
        },
        removeImage: (id) => {
          conversation.releaseDraftImage(id)
          actions.removeImage(id)
        },
        draftImages: ids => conversation.draftImages(ids),
+        releaseSessionImages: (id) => { conversation.releaseSessionImages(id) },
        send: (text, images: readonly ComposerAttachment[], mode) => {
          const trimmed = text.trim()
          if (trimmed === '' && images.length === 0) return
@@ -140,6 +146,9 @@ export function apply(ctx: Context): void {
  slots.register({
    name: 'conversation.empty',
    inject: (): EmptyStateInjected => ({
+      createDraftImages: (files, current) => conversation.createDraftImages(files, current, true),
+      releaseDraftImage: (id) => { conversation.releaseDraftImage(id) },
+      releaseDraftImages: (attachments) => { conversation.releaseDraftImages(attachments) },
      startSession: opts => conversation.startSession(opts),
    }),
  }, EmptyState)
@@ -40,15 +40,13 @@ function ThinkRow({ text, running }: { text: string; running: boolean }) {

 export const AssistantMarkdown = memo(function AssistantMarkdown({ blocks, streaming, interrupted, loadImage = unavailableImage }: AssistantMarkdownProps) {
  const last = blocks.length - 1
-  const images = blocks.filter((block): block is Extract<AssistantBlock, { kind: 'image' }> => block.kind === 'image')
  return (
    <div className={css.root} data-streaming={streaming || undefined}>
-      <ImageGallery images={images} load={loadImage} align="start" />
      {blocks.map((block, i) => {
        switch (block.kind) {
          case 'text': return <MessageText key={i} text={block.text} />
          case 'reasoning': return <ThinkRow key={i} text={block.text} running={streaming && i === last} />
-          case 'image': return null
+          case 'image': return <ImageGallery key={i} images={[block]} load={loadImage} align="start" />
          // Tool-call heads render as tool rows in the chat view's grouping pass.
          case 'tool-call': return null
          default: return <JsonBlock key={i} label="未知内容块" payload={block.block} />
@@ -14,6 +14,7 @@ import type { SelectionTarget, ViewEntry } from './views.ts'

 /** Browser-owned image that has not crossed the durable host boundary. */
 export interface ComposerAttachment {
+  kind: 'image'
  id: string
  file: File
  previewUrl: string
@@ -37,11 +38,13 @@ export interface ConversationInjected {
    version(): number
  }
  /** Create browser previews and append their ids through the declared store action. */
-  addImages(files: readonly File[]): void
+  addImages(files: readonly File[], current: readonly ComposerAttachment[]): string | null
  /** Release one browser preview and remove its id through the declared store action. */
  removeImage(id: string): void
  /** Resolve ordered store ids to the browser-owned draft attachments still available this runtime. */
  draftImages(ids: readonly string[]): readonly ComposerAttachment[]
+  /** Release historical image URLs when this rendered session scope unmounts. */
+  releaseSessionImages(sessionId: SessionId): void
  /** Send choreography: trims, clears the draft optimistically, restores it on failure. */
  send(text: string, images: readonly ComposerAttachment[], mode: 'queue' | 'steer'): void
  /** Cancel the in-flight turn (failure surfaces via snapshot.promptError). */
@@ -72,6 +75,12 @@ export type DetailsSlotProps = PropsRuntime<'details'> & PropsStore<ChatStore> &

 /** Injected share of the no-session empty-state slot. */
 export interface EmptyStateInjected {
+  /** Create service-owned image previews after host-capability preflight. */
+  createDraftImages(files: readonly File[], current: readonly ComposerAttachment[]): readonly ComposerAttachment[]
+  /** Release one service-owned image preview. */
+  releaseDraftImage(id: string): void
+  /** Release all service-owned image previews held by the empty state. */
+  releaseDraftImages(attachments: readonly ComposerAttachment[]): void
  /** The create → navigate → first-send chain, in one service call. */
  startSession(opts: {
    cwd?: string
@@ -29,6 +29,7 @@ import type { ComposerAttachment } from './contract/slots.ts'

 /** Opaque wrapper keeps browser `File` internals outside persisted store state. */
 class BrowserDraftAttachment implements ComposerAttachment {
+  readonly kind = 'image' as const
  readonly id: string
  readonly previewUrl: string
  readonly #file: File
@@ -53,10 +54,17 @@ interface ViewsState {
  listeners: Set<() => void>
 }

+interface ImageUrlEntry {
+  readonly sessionId: SessionId
+  readonly generation: number
+  readonly pending: Promise<string>
+}
+
 /** Scope-addressed conversation service (root singleton, provided as `conversation`). */
 export class ConversationService extends Service {
  private readonly draftAttachments = new Map<string, BrowserDraftAttachment>()
-  private readonly imageUrls = new Map<string, Promise<string>>()
+  private readonly imageUrls = new Map<string, ImageUrlEntry>()
+  private readonly imageGenerations = new Map<SessionId, number>()
  private readonly createdImageUrls = new Set<string>()
  private readonly viewsState: ViewsState = {
    entries: new Map(), cache: null, tick: 0, listeners: new Set(),
@@ -73,6 +81,7 @@ export class ConversationService extends Service {
      this.createdImageUrls.clear()
      this.draftAttachments.clear()
      this.imageUrls.clear()
+      this.imageGenerations.clear()
    }, 'conversation attachment URL cache')
  }

@@ -86,6 +95,7 @@ export class ConversationService extends Service {
   */
  async send(text: string, mode: 'queue' | 'steer', images: readonly File[] = []): Promise<void> {
    const session = this.scopedSession('send')
+    this.validateImages(images, [])
    const uploaded = await Promise.all(images.map(async file => ({
      type: 'image' as const,
      mediaType: imageMediaType(file.type),
@@ -100,9 +110,16 @@ export class ConversationService extends Service {
  /**
   * Create runtime-only draft attachments and their object URLs.
   * @param files - browser-owned image files.
+   * @param current - images already present in the same composer.
+   * @param checkDefaultModel - whether to apply `host.describe`'s default-model capability, used only before a session exists.
   * @returns ordered attachment descriptors whose ids may enter the chat store.
   */
-  createDraftImages(files: readonly File[]): readonly ComposerAttachment[] {
+  createDraftImages(
+    files: readonly File[],
+    current: readonly ComposerAttachment[] = [],
+    checkDefaultModel = false,
+  ): readonly ComposerAttachment[] {
+    this.validateImages(files, current, checkDefaultModel)
    return files.map((file) => {
      const attachment = new BrowserDraftAttachment(file)
      this.draftAttachments.set(attachment.id, attachment)
@@ -154,7 +171,8 @@ export class ConversationService extends Service {
  resolveImage(sessionId: SessionId, attachment: ImageAttachmentRef): Promise<string> {
    const key = `${sessionId}:${attachment.attachmentId}`
    const cached = this.imageUrls.get(key)
-    if (cached !== undefined) return cached
+    if (cached !== undefined) return cached.pending
+    const generation = this.imageGenerations.get(sessionId) ?? 0
    const pending = this.requireSessions().manager.get(sessionId).readAttachment(attachment.attachmentId)
      .then((result) => {
        if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`)
@@ -163,17 +181,39 @@ export class ConversationService extends Service {
        }
        const bytes = Uint8Array.from(result.value.data)
        const url = URL.createObjectURL(new Blob([bytes.buffer], { type: result.value.attachment.mediaType }))
+        if ((this.imageGenerations.get(sessionId) ?? 0) !== generation) {
+          revokePreview(url)
+          throw new Error('historical image scope was released before loading completed')
+        }
        this.createdImageUrls.add(url)
        return url
      })
      .catch((error: unknown) => {
-        this.imageUrls.delete(key)
+        if (this.imageUrls.get(key)?.generation === generation) this.imageUrls.delete(key)
        throw error
      })
-    this.imageUrls.set(key, pending)
+    this.imageUrls.set(key, { sessionId, generation, pending })
    return pending
  }

+  /**
+   * Release every historical image URL owned by one rendered session.
+   * @param sessionId - session whose rendered image scope is ending.
+   */
+  releaseSessionImages(sessionId: SessionId): void {
+    this.imageGenerations.set(sessionId, (this.imageGenerations.get(sessionId) ?? 0) + 1)
+    for (const [key, entry] of this.imageUrls) {
+      if (entry.sessionId !== sessionId) continue
+      this.imageUrls.delete(key)
+      void entry.pending.then((url) => {
+        if (!this.createdImageUrls.delete(url)) return
+        revokePreview(url)
+      }, () => {
+        // A failed or generation-invalidated load owns no cached object URL.
+      })
+    }
+  }
+
  /** Cancel the scoped session's in-flight turn (failures land in promptError and reject, as in send). */
  async cancel(): Promise<void> {
    const session = this.scopedSession('cancel')
@@ -284,6 +324,38 @@ export class ConversationService extends Service {
    if (sessions === undefined) throw new Error('conversation: sessions service unavailable')
    return sessions
  }
+
+  /** Apply host-advertised fast-path checks before any object URL or base64 allocation. */
+  private validateImages(
+    files: readonly File[],
+    current: readonly ComposerAttachment[],
+    checkDefaultModel = false,
+  ): void {
+    const description = this.requireSessions().hostDescription()
+    const modalities = description?.activeModel?.inputModalities
+    if (checkDefaultModel && modalities !== undefined && !modalities.includes('image')) {
+      throw new Error('当前模型不支持图片输入')
+    }
+    const limits = description?.imageLimits
+    const all = [...current.map(attachment => attachment.file), ...files]
+    if (limits !== undefined && all.length > limits.maxImagesPerMessage) {
+      throw new Error(`每条消息最多添加 ${limits.maxImagesPerMessage} 张图片`)
+    }
+    let totalBytes = 0
+    for (const file of all) {
+      const mediaType = imageMediaType(file.type)
+      if (limits !== undefined && !limits.mediaTypes.includes(mediaType)) {
+        throw new Error(`当前部署不支持 ${mediaType} 图片`)
+      }
+      if (limits !== undefined && file.size > limits.maxImageBytes) {
+        throw new Error(`图片 ${file.name || '未命名图片'} 超过单张大小限制`)
+      }
+      totalBytes += file.size
+    }
+    if (limits !== undefined && totalBytes > limits.maxMessageImageBytes) {
+      throw new Error('图片总大小超过单条消息限制')
+    }
+  }
 }

 function bumpViews(state: ViewsState): void {
@@ -36,7 +36,8 @@ function deriveAncestry(list: SessionListState, id: SessionId): readonly Session

 export function ConversationRoot({
  sessionId, useSession, useSessions, useStore, actions,
-  views, addImages, removeImage, draftImages, send, stop, openDetails, loadOlder, open,
+  views, addImages, removeImage, draftImages, releaseSessionImages,
+  send, stop, openDetails, loadOlder, open,
 }: ConversationRootProps) {
  useSyncExternalStore(views.subscribe, views.version)
  const list = views.list()
@@ -63,6 +64,10 @@ export function ConversationRoot({
    }
  }, [actions, attachments, imageIds])

+  useEffect(() => () => {
+    releaseSessionImages(sessionId)
+  }, [releaseSessionImages, sessionId])
+
  const error: InputBarError | null = promptError === null
    ? null
    : { op: promptError.op, message: `${promptError.error.message}（${promptError.error.code}）` }
@@ -145,7 +150,7 @@ export function ConversationRoot({
        error={error}
        variant="composer"
        onDraftChange={actions.setDraft}
-        onAddImages={addImages}
+        onAddImages={files => addImages(files, attachments)}
        onRemoveAttachment={removeImage}
        onSend={(mode) => { send(draft, attachments, mode) }}
        onStop={stop}
@@ -30,7 +30,13 @@ function deriveCwds(state: SessionListState): readonly string[] {
  return [...seen]
 }

-export function EmptyState({ useSessions, startSession }: EmptyStateProps) {
+export function EmptyState({
+  useSessions,
+  createDraftImages,
+  releaseDraftImage,
+  releaseDraftImages,
+  startSession,
+}: EmptyStateProps) {
  const list = useSessions(s => s)
  const cwds = useMemo(() => deriveCwds(list), [list])
  // Local viewing state: the empty state owns no session, so its draft is
@@ -67,21 +73,22 @@ export function EmptyState({ useSessions, startSession }: EmptyStateProps) {
  }

  useEffect(() => () => {
-    for (const attachment of attachmentsRef.current) URL.revokeObjectURL(attachment.previewUrl)
-  }, [])
+    releaseDraftImages(attachmentsRef.current)
+  }, [releaseDraftImages])

-  const addImages = (files: readonly File[]): void => {
-    setAttachments(current => [...current, ...files.map(file => ({
-      id: crypto.randomUUID(), file, previewUrl: URL.createObjectURL(file),
-    }))])
+  const addImages = (files: readonly File[]): string | null => {
+    try {
+      const added = createDraftImages(files, attachments)
+      setAttachments(current => [...current, ...added])
+      return null
+    } catch (reason: unknown) {
+      return reason instanceof Error ? reason.message : String(reason)
+    }
  }

  const removeImage = (id: string): void => {
-    setAttachments((current) => {
-      const removed = current.find(item => item.id === id)
-      if (removed !== undefined) URL.revokeObjectURL(removed.previewUrl)
-      return current.filter(item => item.id !== id)
-    })
+    releaseDraftImage(id)
+    setAttachments(current => current.filter(item => item.id !== id))
  }

  const picker = (
@@ -12,12 +12,6 @@ import type { ComposerAttachment } from '../contract/slots.ts'
 import { ImageLightbox } from './ImageLightbox.tsx'
 import css from './InputBar.module.css'

-const IMAGE_MEDIA_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif'])
-
-function supportedImages(files: Iterable<File>): File[] {
-  return [...files].filter(file => IMAGE_MEDIA_TYPES.has(file.type))
-}
-
 /** Prompt failure surface (mirrors the session snapshot's promptError shape). */
 export interface InputBarError {
  op: 'send' | 'stop'
@@ -36,7 +30,7 @@ export interface InputBarProps {
  /** Optional leading accessory row content (the empty state mounts its cwd picker here). */
  accessory?: ReactNode
  onDraftChange: (text: string) => void
-  onAddImages?: (files: readonly File[]) => void
+  onAddImages?: (files: readonly File[]) => string | null
  onRemoveAttachment?: (id: string) => void
  onSend: (mode: 'queue' | 'steer') => void
  onStop: () => void
@@ -44,7 +38,7 @@ export interface InputBarProps {

 export function InputBar({
  draft, attachments = [], running, disabled, error, variant, placeholder, accessory,
-  onDraftChange, onAddImages = () => {}, onRemoveAttachment = () => {}, onSend, onStop,
+  onDraftChange, onAddImages = () => null, onRemoveAttachment = () => {}, onSend, onStop,
 }: InputBarProps) {
  const empty = draft.trim() === '' && attachments.length === 0
  const [preview, setPreview] = useState<ComposerAttachment | null>(null)
@@ -90,13 +84,12 @@ export function InputBar({

  const onPaste = (event: ClipboardEvent<HTMLTextAreaElement>): void => {
    const files = [...event.clipboardData.items]
-      .filter(item => item.kind === 'file' && IMAGE_MEDIA_TYPES.has(item.type))
+      .filter(item => item.kind === 'file')
      .map(item => item.getAsFile())
      .filter((file): file is File => file !== null)
    if (files.length === 0) return
-    event.preventDefault()
-    setDropError(null)
-    onAddImages(files)
+    if (event.clipboardData.getData('text/plain') === '') event.preventDefault()
+    setDropError(onAddImages(files))
  }

  const onDragEnter = (event: DragEvent<HTMLDivElement>): void => {
@@ -127,13 +120,8 @@ export function InputBar({
    setDragActive(false)
    if (locked) return
    const dropped = [...event.dataTransfer.files]
-    const images = supportedImages(dropped)
-    if (images.length === 0) {
-      setDropError('暂仅支持 PNG、JPEG、WebP 和 GIF 图片')
-      return
-    }
-    setDropError(images.length === dropped.length ? null : '已忽略不受支持的非图片文件')
-    onAddImages(images)
+    if (dropped.length === 0) return
+    setDropError(onAddImages(dropped))
  }

  const closePreview = useCallback(() => { setPreview(null) }, [])
@@ -187,7 +175,10 @@ export function InputBar({
                  type="button"
                  className={css.remove}
                  aria-label={`移除图片 ${attachment.file.name || ''}`}
-                  onClick={() => { onRemoveAttachment(attachment.id) }}
+                  onClick={() => {
+                    setDropError(null)
+                    onRemoveAttachment(attachment.id)
+                  }}
                >×</button>
              </div>
            ))}
@@ -204,7 +195,10 @@ export function InputBar({
            disabled={locked}
            placeholder={placeholder ?? (disabled ? '会话不可用' : running ? '回复生成中，可停止后再输入' : '输入消息，Enter 发送，Shift+Enter 换行')}
            rows={2}
-            onChange={(e) => onDraftChange(e.target.value)}
+            onChange={(e) => {
+              setDropError(null)
+              onDraftChange(e.target.value)
+            }}
            onKeyDown={onKeyDown}
            onPaste={onPaste}
            onCompositionStart={onCompositionStart}