Add web multimodal image attachments

2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions
@@ -5,6 +5,9 @@
 import { z } from 'zod'
 import type { RequestPayload, ResponseValue } from './rpc-map.ts'
 import type { Wire } from './rpc.schema.ts'
+import { imageMediaTypeSchema } from './sessions.schema.ts'
+
+const modalitySchema = z.union([z.literal('text'), z.literal('image')])

 /** host.describe request payload (empty object literal). */
 export const hostDescribeRequestSchema = z.object({}) satisfies z.ZodType<Wire<RequestPayload<'host.describe'>>>
@@ -15,5 +18,20 @@ export const hostDescribeValueSchema = z.object({
  cwd: z.string(),
  provider: z.string().optional(),
  model: z.string().optional(),
+  activeModel: z.object({
+    provider: z.string(),
+    id: z.string(),
+    name: z.string(),
+    description: z.string().optional(),
+    inputModalities: z.array(modalitySchema).optional(),
+    outputModalities: z.array(modalitySchema).optional(),
+  }).optional(),
+  imageLimits: z.object({
+    maxImageBytes: z.number().int().positive(),
+    maxImagesPerMessage: z.number().int().positive(),
+    maxMessageImageBytes: z.number().int().positive(),
+    maxImagePixels: z.number().int().positive(),
+    mediaTypes: z.array(imageMediaTypeSchema),
+  }).optional(),
  attachedSessions: z.number().int().nonnegative(),
 }) satisfies z.ZodType<Wire<ResponseValue<'host.describe'>>>
@@ -4,6 +4,8 @@
 */

 import type { RpcRequest, RpcResponse } from './rpc.ts'
+import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment'
+import type { LlmModelInfo } from '@deepseek-ai/dsh-llm/types'

 /** Host-level unary methods. */
 export interface HostApi {
@@ -20,6 +22,10 @@ export interface HostApi {
    cwd: string
    provider?: string
    model?: string
+    /** Catalog entry for the active route; absent means its capabilities are unknown. */
+    activeModel?: LlmModelInfo
+    /** Resolved authoritative image-upload limits. */
+    imageLimits?: ImageAttachmentLimits
    attachedSessions: number
  }>>
 }
@@ -19,7 +19,7 @@ export interface ApiProxy {
 }

 // ---- Domain interfaces and payload entities ----
-export type { HistoryEntry, SessionsApi, SessionSummary } from './sessions.ts'
+export type { HistoryEntry, PromptContentPart, SessionsApi, SessionSummary } from './sessions.ts'
 export type { HostApi } from './host.ts'
 export type { EventsApi, MuxFrame, HostFrame, ToolCallView, ToolEventView, ToolResultView } from './events.ts'
 export type { ApprovalResponsePayload } from './approvals.ts'
@@ -14,6 +14,7 @@ export interface RpcMethodMap {
  'session.create': SessionsApi['create']
  'session.history': SessionsApi['history']
  'session.prompt': SessionsApi['prompt']
+  'session.attachment': SessionsApi['attachment']
  'session.cancel': SessionsApi['cancel']
  'host.describe': HostApi['describe']
 }
@@ -35,6 +35,7 @@ export const rpcErrorSchema: z.ZodType<RpcError> = z.discriminatedUnion('code',
  z.object({ code: z.literal('bad-request'), message: z.string(), details: z.object({ issues: z.array(z.custom<ZodIssue>()) }) }),
  z.object({ code: z.literal('session-not-found'), message: z.string(), details: z.object({ sessionId: z.string() }) }),
  z.object({ code: z.literal('agent-busy'), message: z.string(), details: z.object({ reason: z.string() }) }),
+  z.object({ code: z.literal('attachment-error'), message: z.string(), details: z.object({ reason: z.string() }) }),
  z.object({ code: z.literal('internal'), message: z.string(), details: z.object({}) }),
 ]) as unknown as z.ZodType<RpcError>

@@ -32,6 +32,7 @@ export interface RpcErrorDetailsMap {
  'bad-request': { issues: ZodIssue[] }
  'session-not-found': { sessionId: SessionId }
  'agent-busy': { reason: string }
+  'attachment-error': { reason: string }
  'internal': {}
 }

@@ -11,6 +11,7 @@ import type { RequestPayload, ResponseValue } from './rpc-map.ts'
 import type { Wire } from './rpc.schema.ts'
 import type { HistoryEntry, SessionSummary } from './sessions.ts'
 import type { ToolEventView } from './events.ts'
+import type { AttachmentIdType, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'

 /** SessionId: one brand cast after shape validation (the only cast point in this domain). */
 export const sessionIdSchema = z.string().min(1) as unknown as z.ZodType<SessionId>
@@ -84,14 +85,25 @@ export const sessionHistoryValueSchema = z.object({
  hasMore: z.boolean(),
 }) satisfies z.ZodType<Wire<ResponseValue<'session.history'>>>

-/** ContentBlock passthrough: core is merge-extensible — the type discriminant envelope is strict, the rest stays wide. */
-export const contentBlockSchema = z.looseObject({ type: z.string() })
+/** Raster image media types accepted by the version-one browser wire. */
+export const imageMediaTypeSchema = z.union([
+  z.literal('image/png'),
+  z.literal('image/jpeg'),
+  z.literal('image/webp'),
+  z.literal('image/gif'),
+])
+
+/** Prompt wire content is intentionally narrower than merge-extensible durable core content. */
+export const promptContentPartSchema = z.discriminatedUnion('type', [
+  z.object({ type: z.literal('text'), text: z.string() }),
+  z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional() }),
+])

 /** session.prompt request payload. */
 export const sessionPromptRequestSchema = z.object({
  sessionId: sessionIdSchema,
  mode: z.union([z.literal('queue'), z.literal('steer')]),
-  content: z.array(contentBlockSchema),
+  content: z.array(promptContentPartSchema),
 }) as unknown as z.ZodType<RequestPayload<'session.prompt'>>

 /** session.prompt response value. */
@@ -99,6 +111,31 @@ export const sessionPromptValueSchema = z.object({
  accepted: z.literal(true),
 }) satisfies z.ZodType<Wire<ResponseValue<'session.prompt'>>>

+/** Opaque attachment id after string-shape validation. */
+export const attachmentIdSchema = z.string().min(1) as unknown as z.ZodType<AttachmentIdType>
+
+/** Durable image reference returned from the authenticated session lookup. */
+export const imageAttachmentRefSchema = z.object({
+  attachmentId: attachmentIdSchema,
+  mediaType: imageMediaTypeSchema,
+  bytes: z.number().int().positive(),
+  width: z.number().int().positive(),
+  height: z.number().int().positive(),
+  name: z.string().optional(),
+}) as unknown as z.ZodType<ImageAttachmentRef>
+
+/** session.attachment request payload. */
+export const sessionAttachmentRequestSchema = z.object({
+  sessionId: sessionIdSchema,
+  attachmentId: attachmentIdSchema,
+}) satisfies z.ZodType<Wire<RequestPayload<'session.attachment'>>>
+
+/** session.attachment response value. */
+export const sessionAttachmentValueSchema = z.object({
+  attachment: imageAttachmentRefSchema,
+  data: z.string(),
+}) satisfies z.ZodType<Wire<ResponseValue<'session.attachment'>>>
+
 /** session.cancel request payload. */
 export const sessionCancelRequestSchema = z.object({
  sessionId: sessionIdSchema,
@@ -4,7 +4,7 @@
 * else references RequestPayload<'session.*'> / ResponseValue<'session.*'>.
 */

-import type { ContentBlock } from '@deepseek-ai/dsh-llm/types'
+import type { AttachmentIdType, ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment'
 import type { SessionEvent, SessionId } from '@deepseek-ai/dsh-session/types'
 import type { RpcId, RpcRequest, RpcResponse } from './rpc.ts'
 import type { ToolEventView } from './events.ts'
@@ -44,6 +44,11 @@ export interface SessionSummary {
  cwd?: string
 }

+/** Browser-submitted prompt content; image bytes are promoted to durable references by the host. */
+export type PromptContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image'; mediaType: ImageMediaType; data: string; name?: string }
+
 /** Session-domain unary methods (the map keys session.* of RpcMethodMap). */
 export interface SessionsApi {
  /** Lists persisted sessions (updatedAt descending). v1 returns everything; cursor is a reserved seat, unimplemented. */
@@ -64,10 +69,14 @@ export interface SessionsApi {
  history(request: RpcRequest<{ sessionId: SessionId; beforeSeq?: number; maxMessages?: number }>):
  Promise<RpcResponse<{ events: HistoryEntry[]; hasMore: boolean }>>

-  /** Sends a message. content is core's ContentBlock[] verbatim; mode maps 1:1 — queue→send, steer→steer. */
-  prompt(request: RpcRequest<{ sessionId: SessionId; mode: 'queue' | 'steer'; content: ContentBlock[] }>):
+  /** Sends text plus temporary base64 image uploads; the host persists images before calling the agent. */
+  prompt(request: RpcRequest<{ sessionId: SessionId; mode: 'queue' | 'steer'; content: PromptContentPart[] }>):
  Promise<RpcResponse<{ accepted: true }>>

+  /** Reads one durable image after proving that this session's log references its id. */
+  attachment(request: RpcRequest<{ sessionId: SessionId; attachmentId: AttachmentIdType }> ):
+  Promise<RpcResponse<{ attachment: ImageAttachmentRef; data: string }>>
+
  /** Stops: clears both FIFOs + aborts the current step (1:1 with agent.cancel). */
  cancel(request: RpcRequest<{ sessionId: SessionId }>): Promise<RpcResponse<{ accepted: true }>>
 }
@@ -16,6 +16,7 @@ import { hostFrameSchema, muxFrameSchema } from '../api/events.schema.ts'
 import { hostDescribeValueSchema } from '../api/host.schema.ts'
 import {
  sessionCancelValueSchema,
+  sessionAttachmentValueSchema,
  sessionCreateValueSchema,
  sessionHistoryValueSchema,
  sessionListValueSchema,
@@ -43,6 +44,7 @@ export interface IApiClient {
    create(payload: RequestPayload<'session.create'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.create'>>>
    history(payload: RequestPayload<'session.history'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.history'>>>
    prompt(payload: RequestPayload<'session.prompt'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.prompt'>>>
+    attachment(payload: RequestPayload<'session.attachment'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.attachment'>>>
    cancel(payload: RequestPayload<'session.cancel'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.cancel'>>>
  }
  host: {
@@ -65,6 +67,7 @@ const UNARY_VALUE_SCHEMAS: { [K in keyof RpcMethodMap]: z.ZodType<Wire<ResponseV
  'session.create': sessionCreateValueSchema,
  'session.history': sessionHistoryValueSchema,
  'session.prompt': sessionPromptValueSchema,
+  'session.attachment': sessionAttachmentValueSchema,
  'session.cancel': sessionCancelValueSchema,
  'host.describe': hostDescribeValueSchema,
 }
@@ -246,6 +249,7 @@ export abstract class AbstractApiClient implements IApiClient {
    create: (payload, signal) => this.callUnary('session.create', payload, signal),
    history: (payload, signal) => this.callUnary('session.history', payload, signal),
    prompt: (payload, signal) => this.callUnary('session.prompt', payload, signal),
+    attachment: (payload, signal) => this.callUnary('session.attachment', payload, signal),
    cancel: (payload, signal) => this.callUnary('session.cancel', payload, signal),
  }

@@ -16,6 +16,7 @@ import type { Wire } from '../api/rpc.schema.ts'
 import { clientRequestSchema, clientResponseSchema } from '../api/rpc.schema.ts'
 import {
  sessionCancelRequestSchema,
+  sessionAttachmentRequestSchema,
  sessionCreateRequestSchema,
  sessionHistoryRequestSchema,
  sessionListRequestSchema,
@@ -42,6 +43,7 @@ const UNARY_ROUTES: UnaryRoutes = {
  'session.create': { schema: sessionCreateRequestSchema, invoke: (api, r) => api.sessions.create(r) },
  'session.history': { schema: sessionHistoryRequestSchema, invoke: (api, r) => api.sessions.history(r) },
  'session.prompt': { schema: sessionPromptRequestSchema, invoke: (api, r) => api.sessions.prompt(r) },
+  'session.attachment': { schema: sessionAttachmentRequestSchema, invoke: (api, r) => api.sessions.attachment(r) },
  'session.cancel': { schema: sessionCancelRequestSchema, invoke: (api, r) => api.sessions.cancel(r) },
  'host.describe': { schema: hostDescribeRequestSchema, invoke: (api, r) => api.host.describe(r) },
 }