feat(vision): add the configurable HTTP vision provider

Registers an OpenAI-compatible chat-completions provider into ctx.imageRecognition with a user-editable baseURL + API key via a settings section and the credential plane. Logs the secret-free vision request body as the image-recognition/llm-request session event. Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-14 10:56:10 +08:00
parent fc180997c6
commit 6409636afb
7 changed files with 485 additions and 0 deletions
@@ -61,4 +61,5 @@ export const KNOWN_SESSION_EVENT_TYPES: ReadonlySet<string> = new Set([
  'turn/start',
  'user/message',
  'web/deepseek-search-llm-request',
+  'image-recognition/llm-request',
 ])
@@ -0,0 +1,47 @@
+{
+  "name": "@deepseek-ai/dsh-image-recognition-http",
+  "description": "Configurable HTTP vision provider for the DeepSeek Harness image-recognition seam — calls a user-supplied OpenAI-compatible chat-completions endpoint with baseURL + API key",
+  "version": "0.1.0-rc.5",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
+    "directory": "packages/vision/image-recognition-http"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./invariant": {
+      "types": "./lib/types/invariant.d.ts",
+      "default": "./lib/invariant.js"
+    },
+    "./src/*": "./src/*",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/index.js",
+    "lib/invariant.js",
+    "lib/types/**/*.d.ts"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@deepseek-ai/schemastery": "workspace:^"
+  },
+  "devDependencies": {
+    "@deepseek-ai/dsh-agent": "workspace:^",
+    "@deepseek-ai/dsh-credentials": "workspace:^",
+    "@deepseek-ai/dsh-image-recognition": "workspace:^",
+    "@deepseek-ai/dsh-invariants": "workspace:^",
+    "@deepseek-ai/dsh-launch-environment": "workspace:^",
+    "@deepseek-ai/dsh-session": "workspace:^",
+    "@deepseek-ai/dsh-settings": "workspace:^",
+    "@deepseek-ai/cordis": "workspace:^"
+  }
+}
@@ -0,0 +1,118 @@
+/**
+ * Register a configurable HTTP vision provider in `ctx.imageRecognition`. The
+ * user points `baseURL` and a model at an OpenAI-compatible chat-completions
+ * endpoint; the provider encodes the image as a data URL and returns recognized
+ * text. Key and endpoint are user-editable through the settings section.
+ * @module @deepseek-ai/dsh-image-recognition-http
+ */
+
+import type { Context } from '@deepseek-ai/cordis'
+import z from '@deepseek-ai/schemastery'
+import type {} from '@deepseek-ai/dsh-agent'
+import { credentialRef } from '@deepseek-ai/dsh-credentials'
+import { installSettingsSection, settingsNamespace } from '@deepseek-ai/dsh-settings'
+import { launchEnvironmentOf } from '@deepseek-ai/dsh-launch-environment'
+import type {} from '@deepseek-ai/dsh-session'
+import type {} from '@deepseek-ai/dsh-image-recognition'
+import {
+  ImageRecognitionHttpProvider,
+  IMAGE_RECOGNITION_DEFAULT_MAX_TOKENS,
+  IMAGE_RECOGNITION_DEFAULT_MODEL,
+} from './provider.ts'
+import type { ImageRecognitionHttpProviderOptions } from './provider.ts'
+
+export {
+  ImageRecognitionHttpProvider,
+  IMAGE_RECOGNITION_DEFAULT_MAX_TOKENS,
+  IMAGE_RECOGNITION_DEFAULT_MODEL,
+  IMAGE_RECOGNITION_HTTP_PROVIDER_ID,
+} from './provider.ts'
+export type { ImageRecognitionHttpProviderOptions } from './provider.ts'
+
+/** Cordis plugin name used by loader diagnostics. */
+export const name = 'image-recognition-http'
+
+/** The image-recognition seam this provider registers into. */
+export const inject = ['imageRecognition']
+
+const DEFAULT_API_KEY_ENV = 'DEEPSEEK_API_KEY'
+
+/** Plugin config (all optional — `apply` fills env-var and constant defaults). */
+export interface Config {
+  /** Literal API key; prefer {@link apiKeyEnv} so no secret enters configuration files. */
+  apiKey?: string
+  /** Credential reference resolved for each recognition; defaults to `DEEPSEEK_API_KEY`. */
+  apiKeyEnv?: string
+  /** OpenAI-compatible endpoint base; `/chat/completions` is appended. */
+  baseURL?: string
+  /** Vision model name. Defaults to `deepseek-v4-flash`. */
+  model?: string
+  /** Upper bound on generated tokens. Defaults to 2048. */
+  maxTokens?: number
+}
+
+export const Config: z<Config> = z.object({
+  apiKey: z.string().role('secret'),
+  apiKeyEnv: z.string().role('credential-ref').default(DEFAULT_API_KEY_ENV),
+  baseURL: z.string(),
+  model: z.string().default(IMAGE_RECOGNITION_DEFAULT_MODEL),
+  maxTokens: z.number().step(1).min(1).default(IMAGE_RECOGNITION_DEFAULT_MAX_TOKENS),
+})
+
+/** Environment variable naming this provider's endpoint. */
+const BASE_URL_ENV = 'DSH_IMAGE_RECOGNITION_BASE_URL'
+
+/** Settings namespace carrying this provider's endpoint, model, and key reference. */
+export const IMAGE_RECOGNITION_HTTP_SETTINGS_NAMESPACE = settingsNamespace('image-recognition-http')
+
+/**
+ * Project one resolved section into the options the provider serves its next
+ * recognition with. Environment fallbacks stay here rather than in the provider:
+ * every value it reads is already fully defaulted.
+ * @param ctx - plugin context supplying the credential and environment planes.
+ * @param config - the currently authoritative section.
+ * @returns options for one recognition.
+ */
+function resolveOptions(ctx: Context, config: Config): ImageRecognitionHttpProviderOptions {
+  const apiKeyEnv = credentialRef(config.apiKeyEnv ?? DEFAULT_API_KEY_ENV)
+  const literalApiKey = config.apiKey !== undefined && config.apiKey.length > 0
+    ? config.apiKey
+    : undefined
+  return {
+    ...literalApiKey === undefined ? {} : { apiKey: literalApiKey },
+    resolveApiKey: async () => {
+      const credentials = ctx.get('credentials')
+      if (credentials !== undefined) return (await credentials.resolve(apiKeyEnv))?.value
+      // Without the seam the environment is the whole credential plane.
+      const ambient = launchEnvironmentOf(ctx).get(apiKeyEnv)
+      return ambient !== undefined && ambient.value.length > 0 ? ambient.value : undefined
+    },
+    baseURL: config.baseURL
+      ?? launchEnvironmentOf(ctx).get(BASE_URL_ENV)?.value
+      ?? '',
+    model: config.model ?? IMAGE_RECOGNITION_DEFAULT_MODEL,
+    maxTokens: config.maxTokens ?? IMAGE_RECOGNITION_DEFAULT_MAX_TOKENS,
+    recordRequest: (request) => {
+      ctx.get('agents')?.currentInitiator()?.session.append(
+        'image-recognition/llm-request',
+        request,
+      )
+    },
+  }
+}
+
+/** Register the HTTP image-recognition provider with `ctx.imageRecognition`. */
+export function apply(ctx: Context, config: Config): void {
+  let current: () => Config = () => config
+  installSettingsSection(ctx, IMAGE_RECOGNITION_HTTP_SETTINGS_NAMESPACE, Config, config, {
+    setSource: (source) => {
+      current = source
+    },
+    // The registration carries no resolved value: the provider projects the
+    // section per recognition, so a committed change needs no re-registration.
+    onChange: () => {},
+  })
+  ctx.imageRecognition.registerProvider(
+    new ImageRecognitionHttpProvider(() => resolveOptions(ctx, current())),
+  )
+}
@@ -0,0 +1,30 @@
+/**
+ * Package-owned invariant companion for `@deepseek-ai/dsh-image-recognition-http`.
+ * @module @deepseek-ai/dsh-image-recognition-http/invariant
+ */
+
+/* jscpd:ignore-start */
+import type { Context } from '@deepseek-ai/cordis'
+import type { InvariantInstaller } from '@deepseek-ai/dsh-invariants'
+
+const PACKAGE_NAME = '@deepseek-ai/dsh-image-recognition-http'
+
+/** Cordis companion plugin name. */
+export const name = 'image-recognition-http-invariant'
+/** Service required before the companion can reserve package ownership. */
+export const inject = ['invariants']
+
+/**
+ * No runtime invariant: the provider registers into the seam registry and keeps
+ * no observable independent state; HTTP behavior is asserted by its own tests.
+ */
+const install: InvariantInstaller = () => {}
+
+/**
+ * Register this package's invariant companion.
+ * @param ctx - Cordis context carrying the invariant service.
+ * @returns the installed registration's disposer after setup succeeds.
+ */
+export const apply = (ctx: Context): Promise<() => void> =>
+  Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install))
+/* jscpd:ignore-end */
@@ -0,0 +1,176 @@
+/**
+ * An OpenAI-compatible chat-completions vision provider for
+ * `ctx.imageRecognition`. Encodes the image as a data URL and sends it to a
+ * user-configured `baseURL`/`model` endpoint, returning the recognized text.
+ * @module @deepseek-ai/dsh-image-recognition-http/provider
+ */
+
+import { readFile } from 'node:fs/promises'
+import { extname } from 'node:path'
+import {
+  ImageRecognitionError,
+  type ImageInput,
+  type ImageRecognitionProvider,
+  type ImageRecognitionRequest,
+  type ImageRecognitionResult,
+} from '@deepseek-ai/dsh-image-recognition'
+import type {} from '@deepseek-ai/dsh-session/types'
+
+export const IMAGE_RECOGNITION_HTTP_PROVIDER_ID = 'http'
+
+/** Secret-free vision request body recorded before dispatch. */
+export interface ImageRecognitionLlmRequest {
+  readonly model: string
+  readonly max_tokens: number
+  readonly messages: readonly unknown[]
+}
+
+declare module '@deepseek-ai/dsh-session/types' {
+  interface SessionEventMap {
+    /** Secret-free vision request body recorded before dispatch. */
+    'image-recognition/llm-request': ImageRecognitionLlmRequest
+  }
+}
+
+export const IMAGE_RECOGNITION_DEFAULT_MODEL = 'deepseek-v4-flash'
+export const IMAGE_RECOGNITION_DEFAULT_MAX_TOKENS = 2048
+
+const USER_AGENT = 'deepseek-harness/0.0.1'
+
+/** Options the provider serves one recognition with, resolved by the plugin. */
+export interface ImageRecognitionHttpProviderOptions {
+  /** Literal API key, when configured. */
+  readonly apiKey?: string
+  /** Resolve the key from the credential/ambient plane per recognition. */
+  readonly resolveApiKey: () => Promise<string | undefined>
+  /** Endpoint base; `/chat/completions` is appended. */
+  readonly baseURL: string
+  /** Vision model name. */
+  readonly model: string
+  /** Upper bound on generated tokens. */
+  readonly maxTokens: number
+  /** Record the outgoing LLM request for the session log. */
+  readonly recordRequest?: (request: ImageRecognitionLlmRequest) => void
+}
+
+const MEDIA_TYPE_BY_EXTENSION: Readonly<Record<string, string>> = {
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.webp': 'image/webp',
+  '.gif': 'image/gif',
+}
+
+/** Best-effort media type for a file path; undefined when the extension is unknown. */
+function mediaTypeForPath(filePath: string): string | undefined {
+  return MEDIA_TYPE_BY_EXTENSION[extname(filePath).toLowerCase()]
+}
+
+/** Whether the value is a network (fetch) AbortError. */
+function isAbortError(error: unknown): boolean {
+  return error instanceof Error && error.name === 'AbortError'
+}
+
+/** Resolve the request's image to a data URL or plain URL the endpoint accepts. */
+async function imageSource(input: ImageInput): Promise<string> {
+  switch (input.kind) {
+    case 'base64':
+      return `data:${input.mediaType};base64,${input.base64}`
+    case 'url':
+      return input.url
+    case 'file-path': {
+      const mediaType = mediaTypeForPath(input.filePath)
+      if (mediaType === undefined) {
+        throw new ImageRecognitionError(
+          `cannot infer media type for "${input.filePath}"; pass the image as base64 with an explicit media type`,
+          'IMAGE_RECOGNITION_PROVIDER_ERROR',
+        )
+      }
+      const bytes = await readFile(input.filePath)
+      return `data:${mediaType};base64,${bytes.toString('base64')}`
+    }
+  }
+}
+
+/**
+ * A vision endpoint that recognizes image content through the OpenAI-compatible
+ * chat-completions protocol.
+ */
+export class ImageRecognitionHttpProvider implements ImageRecognitionProvider {
+  readonly id = IMAGE_RECOGNITION_HTTP_PROVIDER_ID
+
+  constructor(private readonly resolveOptions: () => ImageRecognitionHttpProviderOptions) {}
+
+  available(): boolean {
+    return URL.canParse(this.resolveOptions().baseURL)
+  }
+
+  async recognize(request: ImageRecognitionRequest, signal?: AbortSignal): Promise<ImageRecognitionResult> {
+    // Snapshot the section once per recognition so live settings changes apply.
+    const options = this.resolveOptions()
+    const apiKey = options.apiKey ?? (await options.resolveApiKey())
+    if (apiKey === undefined || apiKey.length === 0) {
+      throw new ImageRecognitionError(
+        'image-recognition provider has no API key configured; set one in the plugin settings or environment',
+        'IMAGE_RECOGNITION_PROVIDER_CREDENTIAL_MISSING',
+      )
+    }
+    const source = await imageSource(request.image)
+    const payload = {
+      model: options.model,
+      max_tokens: options.maxTokens,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: request.prompt ?? 'Describe the contents of this image.' },
+          { type: 'image_url', image_url: { url: source } },
+        ],
+      }],
+    }
+    options.recordRequest?.(payload)
+
+    let response: Response
+    try {
+      response = await fetch(`${options.baseURL}/chat/completions`, {
+        method: 'POST',
+        redirect: 'error',
+        headers: {
+          'authorization': `Bearer ${apiKey}`,
+          'content-type': 'application/json',
+          'accept': 'application/json',
+          'user-agent': USER_AGENT,
+        },
+        body: JSON.stringify(payload),
+        ...signal !== undefined ? { signal } : {},
+      })
+    } catch (error) {
+      if (isAbortError(error)) {
+        throw new ImageRecognitionError('image recognition aborted', 'IMAGE_RECOGNITION_ABORTED', { cause: error })
+      }
+      throw new ImageRecognitionError(`image recognition network failure: ${String(error)}`, 'IMAGE_RECOGNITION_PROVIDER_ERROR', { cause: error })
+    }
+
+    if (!response.ok) {
+      let detail = ''
+      try {
+        const body = (await response.json()) as { error?: { message?: unknown } }
+        detail = typeof body.error?.message === 'string' ? `: ${body.error.message}` : ''
+      } catch {
+        // Non-JSON error bodies carry no structured detail.
+      }
+      throw new ImageRecognitionError(
+        `image-recognition provider returned ${response.status}${detail}`,
+        'IMAGE_RECOGNITION_PROVIDER_ERROR',
+      )
+    }
+
+    const data = (await response.json()) as {
+      choices?: Array<{ message?: { content?: unknown } }>
+    }
+    const content = data.choices?.[0]?.message?.content
+    if (typeof content !== 'string' || content.length === 0) {
+      throw new ImageRecognitionError('image-recognition provider returned no text', 'IMAGE_RECOGNITION_PROVIDER_ERROR')
+    }
+    return { text: content }
+  }
+}
@@ -0,0 +1,74 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type { ImageRecognitionRequest } from '@deepseek-ai/dsh-image-recognition'
+import {
+  ImageRecognitionHttpProvider,
+  IMAGE_RECOGNITION_DEFAULT_MODEL,
+  type ImageRecognitionHttpProviderOptions,
+} from '../src/provider.ts'
+
+/** A stub vision endpoint. */
+function stubFetch(response: Partial<Response>): void {
+  vi.stubGlobal('fetch', vi.fn(async () => response))
+}
+
+function makeProvider(overrides: Partial<ImageRecognitionHttpProviderOptions> = {}): ImageRecognitionHttpProvider {
+  return new ImageRecognitionHttpProvider(() => ({
+    resolveApiKey: async () => 'key',
+    baseURL: 'https://vision.example.com/v1',
+    model: IMAGE_RECOGNITION_DEFAULT_MODEL,
+    maxTokens: 128,
+    ...overrides,
+  }))
+}
+
+const request: ImageRecognitionRequest = {
+  image: { kind: 'base64', base64: 'aGVsbG8=', mediaType: 'image/png' },
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals()
+})
+
+describe('ImageRecognitionHttpProvider.available', () => {
+  it('is unavailable when no baseURL is configured', () => {
+    expect(makeProvider({ baseURL: '' }).available()).toBe(false)
+  })
+
+  it('is available when a baseURL is configured', () => {
+    expect(makeProvider().available()).toBe(true)
+  })
+})
+
+describe('ImageRecognitionHttpProvider.recognize', () => {
+  it('returns the recognized text from a 200 response', async () => {
+    stubFetch({ ok: true, status: 200, json: async () => ({ choices: [{ message: { content: 'a cat' } }] }) })
+    await expect(makeProvider().recognize(request)).resolves.toEqual({ text: 'a cat' })
+  })
+
+  it('sends the image as a data URL to the configured endpoint', async () => {
+    const fetchMock = vi.fn(async (_url: string, _init?: { body?: string }) => ({ ok: true, status: 200, json: async () => ({ choices: [{ message: { content: 'x' } }] }) }))
+    vi.stubGlobal('fetch', fetchMock)
+    await makeProvider({ baseURL: 'https://v.example.com/v1' }).recognize(request)
+    const [url, init] = fetchMock.mock.calls[0]!
+    expect(url).toBe('https://v.example.com/v1/chat/completions')
+    const body = JSON.parse(init?.body ?? '{}') as { messages: Array<{ content: Array<{ type: string; image_url: { url: string } }> }> }
+    expect(body.messages[0]!.content[1]!.image_url.url).toBe('data:image/png;base64,aGVsbG8=')
+  })
+
+  it('throws IMAGE_RECOGNITION_PROVIDER_CREDENTIAL_MISSING without a key', async () => {
+    await expect(makeProvider({ apiKey: '', resolveApiKey: async () => undefined }).recognize(request))
+      .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_CREDENTIAL_MISSING' }))
+  })
+
+  it('maps a non-2xx response to IMAGE_RECOGNITION_PROVIDER_ERROR', async () => {
+    stubFetch({ ok: false, status: 429, json: async () => ({ error: { message: 'rate limited' } }) })
+    await expect(makeProvider().recognize(request))
+      .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_ERROR' }))
+  })
+
+  it('maps a network failure to IMAGE_RECOGNITION_PROVIDER_ERROR', async () => {
+    vi.stubGlobal('fetch', vi.fn(async () => { throw new TypeError('network down') }))
+    await expect(makeProvider().recognize(request))
+      .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_ERROR' }))
+  })
+})
@@ -0,0 +1,39 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "lib/types"
+  },
+  "include": [
+    "src"
+  ],
+  "references": [
+    {
+      "path": "../../../vendor/cordis"
+    },
+    {
+      "path": "../../../vendor/schemastery"
+    },
+    {
+      "path": "../image-recognition"
+    },
+    {
+      "path": "../../core/agent"
+    },
+    {
+      "path": "../../core/session"
+    },
+    {
+      "path": "../../credentials/credentials"
+    },
+    {
+      "path": "../../settings/settings"
+    },
+    {
+      "path": "../../util/launch-environment"
+    },
+    {
+      "path": "../../runtime-diagnostics/invariants"
+    }
+  ]
+}