feat(vision): add the model-facing recognition consumer

Registers the recognize_image tool and a bundled image-recognition skill, and hooks agent/pre-step to deterministically inject the skill body when a step input carries an image (content block or image path/URL), so the model recognizes the image before continuing the task. Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-14 10:57:38 +08:00
parent 6409636afb
commit 457304e619
5 changed files with 324 additions and 0 deletions
@@ -0,0 +1,47 @@
+{
+  "name": "@deepseek-ai/dsh-tool-image-recognition",
+  "description": "Model-facing image-recognition consumer for the DeepSeek Harness — recognize_image tool, bundled image-recognition skill, and deterministic image-task pre-step injection",
+  "version": "0.1.0-rc.5",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
+    "directory": "packages/vision/tool-image-recognition"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./invariant": {
+      "types": "./lib/types/invariant.d.ts",
+      "default": "./lib/invariant.js"
+    },
+    "./src/*": "./src/*",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/index.js",
+    "lib/invariant.js",
+    "lib/types/**/*.d.ts"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@deepseek-ai/schemastery": "workspace:^"
+  },
+  "devDependencies": {
+    "@deepseek-ai/dsh-agent": "workspace:^",
+    "@deepseek-ai/dsh-image-recognition": "workspace:^",
+    "@deepseek-ai/dsh-invariants": "workspace:^",
+    "@deepseek-ai/dsh-llm": "workspace:^",
+    "@deepseek-ai/dsh-session": "workspace:^",
+    "@deepseek-ai/dsh-skill": "workspace:^",
+    "@deepseek-ai/dsh-tools": "workspace:^",
+    "@deepseek-ai/cordis": "workspace:^"
+  }
+}
@@ -0,0 +1,170 @@
+/**
+ * Model-facing image-recognition consumer. Registers a bundled `image-recognition`
+ * skill and a `recognize_image` tool, and — when a step's input carries an image
+ * — deterministically injects the skill body before the model acts, so the model
+ * recognizes the image first and then continues the task.
+ * @module @deepseek-ai/dsh-tool-image-recognition
+ */
+
+import type { Context } from '@deepseek-ai/cordis'
+import z from '@deepseek-ai/schemastery'
+import type { PreStepDecision } from '@deepseek-ai/dsh-agent'
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import { createUserMessage } from '@deepseek-ai/dsh-llm'
+import type { MessageSource } from '@deepseek-ai/dsh-llm'
+import type { UserMessage } from '@deepseek-ai/dsh-session'
+import { isModelInvocable, renderSkillContent } from '@deepseek-ai/dsh-skill'
+import type { ImageInput, ImageRecognitionResult } from '@deepseek-ai/dsh-image-recognition'
+
+export const name = 'tool-image-recognition'
+export const inject = ['agents', 'imageRecognition', 'skills', 'systemPrompt', 'tools']
+
+const SKILL_NAME = 'image-recognition'
+
+/** The `{kind:'plugin'}` source stamped on every injection this plugin makes. */
+const PLUGIN_SOURCE: MessageSource = { kind: 'plugin', plugin: 'tool-image-recognition' }
+
+/** The bundled recognition skill body, injected verbatim when an image task is detected. */
+const SKILL_CONTENT = [
+  '## Image recognition',
+  'When the task involves an image (an attached image, an image file path, or an image URL), FIRST recognize its content before continuing the task:',
+  '',
+  '1. Determine which image the task refers to from the conversation (attachment, path, or URL).',
+  '2. Call `recognize_image` with that image and, when useful, a `prompt` naming what to extract (e.g. "transcribe the text", "describe the scene", "read the numbers").',
+  '3. Use the recognized text as ground truth to complete the original task.',
+  '',
+  'Do not guess at image contents from a filename or description — run `recognize_image` and act on its result.',
+].join('\n')
+
+/** Plugin config (all optional). */
+export interface Config {
+  /** Inject the skill body when a step input carries an image content block. */
+  detectImageBlocks?: boolean
+  /** Inject the skill body when a step input text names an image file path or URL. */
+  detectImagePaths?: boolean
+}
+
+export const Config: z<Config> = z.object({
+  detectImageBlocks: z.boolean().default(true),
+  detectImagePaths: z.boolean().default(true),
+})
+
+/**
+ * Whether a step input carries an image: an image content block, or a text
+ * block naming an image file path / image URL. Never reads image bytes — it is
+ * a cheap signal for whether to inject the recognition skill.
+ * @param messages - the step's claimed batch.
+ * @param detectBlocks - whether image content blocks count as a signal.
+ * @param detectPaths - whether image file paths / URLs in text count as a signal.
+ * @returns true when any supported image signal is present.
+ */
+export function hasImageSignal(
+  messages: readonly UserMessage[],
+  detectBlocks: boolean,
+  detectPaths: boolean,
+): boolean {
+  return messages.some(message =>
+    message.content.some((block) => {
+      if (block.type === 'image') return detectBlocks
+      if (block.type !== 'text') return false
+      return detectPaths && IMAGE_TEXT_RE.test(block.text)
+    }),
+  )
+}
+
+/** An image file path or image URL named in text. */
+const IMAGE_TEXT_RE = /\.(?:png|jpe?g|webp|gif)\b|https?:\/\/[^\s]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s]*)?/i
+
+/** Normalize a tool `image` argument into the seam's {@link ImageInput}. */
+function toImageInput(image: string): ImageInput {
+  if (image.startsWith('data:')) {
+    const match = /^data:([^;,]+);base64,(.+)$/s.exec(image)
+    if (match === null) throw new Error('recognize_image: invalid data URL')
+    // RegExpExecArray group access types as string | undefined; the anchored
+    // pattern guarantees both groups when it matches.
+    const mediaType = match[1]
+    const base64 = match[2]
+    if (mediaType === undefined || base64 === undefined) throw new Error('recognize_image: invalid data URL')
+    return { kind: 'base64', base64, mediaType }
+  }
+  if (/^https?:\/\//.test(image)) return { kind: 'url', url: image }
+  return { kind: 'file-path', filePath: image }
+}
+
+/**
+ * Install the recognition skill, tool, deterministic injection, and prompt
+ * section.
+ * @param ctx - plugin context.
+ * @param config - validated {@link Config}.
+ */
+export function apply(ctx: Context, config: Config = {}): void {
+  const detectBlocks = config.detectImageBlocks ?? true
+  const detectPaths = config.detectImagePaths ?? true
+
+  ctx.skills.register({
+    name: SKILL_NAME,
+    source: 'runtime',
+    description: 'Recognize and describe the contents of an image (objects, scenes, and text) so you can continue a task about it.',
+    whenToUse: 'Use whenever the task involves an attached, referenced, or named image and you need to know what it contains.',
+    invocation: { modelInvocable: true, userInvocable: true },
+    content: SKILL_CONTENT,
+  })
+
+  const recognizeImageTool = defineTool({
+    name: 'recognize_image',
+    description: 'Recognize the contents of an image (objects, scenes, and text) through the configured image provider and return the recognized text.',
+    parameters: {
+      image: { type: 'string', required: true, description: 'The image to recognize: a file path, an https URL, or a data: URL.' },
+      prompt: { type: 'string', description: 'Optional recognition focus, e.g. "transcribe the text" or "describe the scene".' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          text: { type: 'string', required: true },
+        },
+      },
+      render: (_args, value: ImageRecognitionResult) => [{ type: 'text', text: value.text }],
+    },
+    async execute(args, exec) {
+      const result = await ctx.imageRecognition.recognize({
+        image: toImageInput(args.image),
+        ...args.prompt !== undefined ? { prompt: args.prompt } : {},
+      }, exec.signal)
+      return result
+    },
+    presentCall(args) {
+      return { card: 'generic', title: 'Recognize image', kind: 'read', rawInput: args.image }
+    },
+  })
+  ctx.tools.register(recognizeImageTool)
+
+  // Deterministic injection: when the step input carries an image, prepend the
+  // recognition skill body so the model recognizes it before other actions.
+  // Delegate first, then prepend onto the decision (never veto, never rewrite).
+  ctx.on('agent/pre-step', async (
+    { agent, signal },
+    next,
+  ): Promise<PreStepDecision> => {
+    const decision = await next()
+    if (decision.kind === 'reject') return decision
+    if (!hasImageSignal(decision.messages, detectBlocks, detectPaths)) return decision
+    signal.throwIfAborted()
+    const lookup = { cwd: agent.session.header.cwd, signal, scope: agent }
+    const skill = await ctx.skills.get(SKILL_NAME, lookup)
+    signal.throwIfAborted()
+    if (skill === undefined || !isModelInvocable(skill)) return decision
+    const injection = createUserMessage({
+      content: [{ type: 'text', text: renderSkillContent(skill) }],
+      source: { ...PLUGIN_SOURCE, form: 'instructions', summary: 'recognize image first' },
+    })
+    return { kind: 'enter', messages: [injection, ...decision.messages] }
+  })
+
+  ctx.systemPrompt.section({
+    name: 'tool:image-recognition',
+    order: 115,
+    text: 'When a task involves an image, recognize it first with `recognize_image` before continuing; do not infer image contents from a filename or description.',
+  })
+}
@@ -0,0 +1,31 @@
+/**
+ * Package-owned invariant companion for `@deepseek-ai/dsh-tool-image-recognition`.
+ * @module @deepseek-ai/dsh-tool-image-recognition/invariant
+ */
+
+/* jscpd:ignore-start */
+import type { Context } from '@deepseek-ai/cordis'
+import type { InvariantInstaller } from '@deepseek-ai/dsh-invariants'
+
+const PACKAGE_NAME = '@deepseek-ai/dsh-tool-image-recognition'
+
+/** Cordis companion plugin name. */
+export const name = 'tool-image-recognition-invariant'
+/** Service required before the companion can reserve package ownership. */
+export const inject = ['invariants']
+
+/**
+ * No runtime invariant: the consumer registers into the tools/skills registries
+ * and its injection is asserted by its own tests; it publishes no independent
+ * observation stream.
+ */
+const install: InvariantInstaller = () => {}
+
+/**
+ * Register this package's invariant companion.
+ * @param ctx - Cordis context carrying the invariant service.
+ * @returns the installed registration's disposer after setup succeeds.
+ */
+export const apply = (ctx: Context): Promise<() => void> =>
+  Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install))
+/* jscpd:ignore-end */
@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest'
+import type { UserMessage } from '@deepseek-ai/dsh-session'
+import { hasImageSignal } from '../src/index.ts'
+
+function userMessage(blocks: UserMessage['content']): UserMessage {
+  return { id: 'u', role: 'user', source: { kind: 'user' }, content: blocks } as UserMessage
+}
+
+/** A content block carrying an image signal (the payload is irrelevant here). */
+function imageBlock(): UserMessage['content'][number] {
+  return { type: 'image', attachment: {} as never } as UserMessage['content'][number]
+}
+
+describe('hasImageSignal', () => {
+  it('detects an image content block when block detection is on', () => {
+    expect(hasImageSignal([userMessage([imageBlock()])], true, true)).toBe(true)
+  })
+
+  it('ignores image blocks when block detection is off', () => {
+    expect(hasImageSignal([userMessage([imageBlock()])], false, true)).toBe(false)
+  })
+
+  it('detects an image file path in text when path detection is on', () => {
+    const message = userMessage([{ type: 'text', text: 'transcribe /tmp/screenshot.png' }])
+    expect(hasImageSignal([message], true, true)).toBe(true)
+  })
+
+  it('detects an image URL in text', () => {
+    const message = userMessage([{ type: 'text', text: 'look at https://e.test/photo.JPG' }])
+    expect(hasImageSignal([message], true, true)).toBe(true)
+  })
+
+  it('ignores plain text with no image signal', () => {
+    const message = userMessage([{ type: 'text', text: 'summarize this document' }])
+    expect(hasImageSignal([message], true, true)).toBe(false)
+  })
+})
@@ -0,0 +1,39 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "lib/types"
+  },
+  "include": [
+    "src"
+  ],
+  "references": [
+    {
+      "path": "../../../vendor/cordis"
+    },
+    {
+      "path": "../../../vendor/schemastery"
+    },
+    {
+      "path": "../image-recognition"
+    },
+    {
+      "path": "../../core/agent"
+    },
+    {
+      "path": "../../core/session"
+    },
+    {
+      "path": "../../llm/llm"
+    },
+    {
+      "path": "../../core/tools"
+    },
+    {
+      "path": "../../skill/skill"
+    },
+    {
+      "path": "../../runtime-diagnostics/invariants"
+    }
+  ]
+}