From 457304e619504786e35b61b4b592951905b5d32b Mon Sep 17 00:00:00 2001 From: Pine Date: Fri, 14 Aug 2026 10:57:38 +0800 Subject: [PATCH] feat(vision): add the model-facing recognition consumer Registers the recognize_image tool and a bundled image-recognition skill, and hooks agent/pre-step to deterministically inject the skill body when a step input carries an image (content block or image path/URL), so the model recognizes the image before continuing the task. Co-Authored-By: Claude --- .../tool-image-recognition/package.json | 47 +++++ .../tool-image-recognition/src/index.ts | 170 ++++++++++++++++++ .../tool-image-recognition/src/invariant.ts | 31 ++++ .../tests/has-image-signal.spec.ts | 37 ++++ .../tool-image-recognition/tsconfig.json | 39 ++++ 5 files changed, 324 insertions(+) create mode 100644 packages/vision/tool-image-recognition/package.json create mode 100644 packages/vision/tool-image-recognition/src/index.ts create mode 100644 packages/vision/tool-image-recognition/src/invariant.ts create mode 100644 packages/vision/tool-image-recognition/tests/has-image-signal.spec.ts create mode 100644 packages/vision/tool-image-recognition/tsconfig.json diff --git a/packages/vision/tool-image-recognition/package.json b/packages/vision/tool-image-recognition/package.json new file mode 100644 index 0000000000..6fff441e33 --- /dev/null +++ b/packages/vision/tool-image-recognition/package.json @@ -0,0 +1,47 @@ +{ + "name": "@deepseek-ai/dsh-tool-image-recognition", + "description": "Model-facing image-recognition consumer for the DeepSeek Harness — recognize_image tool, bundled image-recognition skill, and deterministic image-task pre-step injection", + "version": "0.1.0-rc.5", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", + "directory": "packages/vision/tool-image-recognition" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./invariant": { + "types": "./lib/types/invariant.d.ts", + "default": "./lib/invariant.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/invariant.js", + "lib/types/**/*.d.ts" + ], + "license": "MIT", + "dependencies": { + "@deepseek-ai/schemastery": "workspace:^" + }, + "devDependencies": { + "@deepseek-ai/dsh-agent": "workspace:^", + "@deepseek-ai/dsh-image-recognition": "workspace:^", + "@deepseek-ai/dsh-invariants": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-skill": "workspace:^", + "@deepseek-ai/dsh-tools": "workspace:^", + "@deepseek-ai/cordis": "workspace:^" + } +} diff --git a/packages/vision/tool-image-recognition/src/index.ts b/packages/vision/tool-image-recognition/src/index.ts new file mode 100644 index 0000000000..557f1bf776 --- /dev/null +++ b/packages/vision/tool-image-recognition/src/index.ts @@ -0,0 +1,170 @@ +/** + * Model-facing image-recognition consumer. Registers a bundled `image-recognition` + * skill and a `recognize_image` tool, and — when a step's input carries an image + * — deterministically injects the skill body before the model acts, so the model + * recognizes the image first and then continues the task. + * @module @deepseek-ai/dsh-tool-image-recognition + */ + +import type { Context } from '@deepseek-ai/cordis' +import z from '@deepseek-ai/schemastery' +import type { PreStepDecision } from '@deepseek-ai/dsh-agent' +import { defineTool } from '@deepseek-ai/dsh-tools' +import { createUserMessage } from '@deepseek-ai/dsh-llm' +import type { MessageSource } from '@deepseek-ai/dsh-llm' +import type { UserMessage } from '@deepseek-ai/dsh-session' +import { isModelInvocable, renderSkillContent } from '@deepseek-ai/dsh-skill' +import type { ImageInput, ImageRecognitionResult } from '@deepseek-ai/dsh-image-recognition' + +export const name = 'tool-image-recognition' +export const inject = ['agents', 'imageRecognition', 'skills', 'systemPrompt', 'tools'] + +const SKILL_NAME = 'image-recognition' + +/** The `{kind:'plugin'}` source stamped on every injection this plugin makes. */ +const PLUGIN_SOURCE: MessageSource = { kind: 'plugin', plugin: 'tool-image-recognition' } + +/** The bundled recognition skill body, injected verbatim when an image task is detected. */ +const SKILL_CONTENT = [ + '## Image recognition', + 'When the task involves an image (an attached image, an image file path, or an image URL), FIRST recognize its content before continuing the task:', + '', + '1. Determine which image the task refers to from the conversation (attachment, path, or URL).', + '2. Call `recognize_image` with that image and, when useful, a `prompt` naming what to extract (e.g. "transcribe the text", "describe the scene", "read the numbers").', + '3. Use the recognized text as ground truth to complete the original task.', + '', + 'Do not guess at image contents from a filename or description — run `recognize_image` and act on its result.', +].join('\n') + +/** Plugin config (all optional). */ +export interface Config { + /** Inject the skill body when a step input carries an image content block. */ + detectImageBlocks?: boolean + /** Inject the skill body when a step input text names an image file path or URL. */ + detectImagePaths?: boolean +} + +export const Config: z = z.object({ + detectImageBlocks: z.boolean().default(true), + detectImagePaths: z.boolean().default(true), +}) + +/** + * Whether a step input carries an image: an image content block, or a text + * block naming an image file path / image URL. Never reads image bytes — it is + * a cheap signal for whether to inject the recognition skill. + * @param messages - the step's claimed batch. + * @param detectBlocks - whether image content blocks count as a signal. + * @param detectPaths - whether image file paths / URLs in text count as a signal. + * @returns true when any supported image signal is present. + */ +export function hasImageSignal( + messages: readonly UserMessage[], + detectBlocks: boolean, + detectPaths: boolean, +): boolean { + return messages.some(message => + message.content.some((block) => { + if (block.type === 'image') return detectBlocks + if (block.type !== 'text') return false + return detectPaths && IMAGE_TEXT_RE.test(block.text) + }), + ) +} + +/** An image file path or image URL named in text. */ +const IMAGE_TEXT_RE = /\.(?:png|jpe?g|webp|gif)\b|https?:\/\/[^\s]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s]*)?/i + +/** Normalize a tool `image` argument into the seam's {@link ImageInput}. */ +function toImageInput(image: string): ImageInput { + if (image.startsWith('data:')) { + const match = /^data:([^;,]+);base64,(.+)$/s.exec(image) + if (match === null) throw new Error('recognize_image: invalid data URL') + // RegExpExecArray group access types as string | undefined; the anchored + // pattern guarantees both groups when it matches. + const mediaType = match[1] + const base64 = match[2] + if (mediaType === undefined || base64 === undefined) throw new Error('recognize_image: invalid data URL') + return { kind: 'base64', base64, mediaType } + } + if (/^https?:\/\//.test(image)) return { kind: 'url', url: image } + return { kind: 'file-path', filePath: image } +} + +/** + * Install the recognition skill, tool, deterministic injection, and prompt + * section. + * @param ctx - plugin context. + * @param config - validated {@link Config}. + */ +export function apply(ctx: Context, config: Config = {}): void { + const detectBlocks = config.detectImageBlocks ?? true + const detectPaths = config.detectImagePaths ?? true + + ctx.skills.register({ + name: SKILL_NAME, + source: 'runtime', + description: 'Recognize and describe the contents of an image (objects, scenes, and text) so you can continue a task about it.', + whenToUse: 'Use whenever the task involves an attached, referenced, or named image and you need to know what it contains.', + invocation: { modelInvocable: true, userInvocable: true }, + content: SKILL_CONTENT, + }) + + const recognizeImageTool = defineTool({ + name: 'recognize_image', + description: 'Recognize the contents of an image (objects, scenes, and text) through the configured image provider and return the recognized text.', + parameters: { + image: { type: 'string', required: true, description: 'The image to recognize: a file path, an https URL, or a data: URL.' }, + prompt: { type: 'string', description: 'Optional recognition focus, e.g. "transcribe the text" or "describe the scene".' }, + }, + output: { + schema: { + type: 'object', + additionalProperties: false, + properties: { + text: { type: 'string', required: true }, + }, + }, + render: (_args, value: ImageRecognitionResult) => [{ type: 'text', text: value.text }], + }, + async execute(args, exec) { + const result = await ctx.imageRecognition.recognize({ + image: toImageInput(args.image), + ...args.prompt !== undefined ? { prompt: args.prompt } : {}, + }, exec.signal) + return result + }, + presentCall(args) { + return { card: 'generic', title: 'Recognize image', kind: 'read', rawInput: args.image } + }, + }) + ctx.tools.register(recognizeImageTool) + + // Deterministic injection: when the step input carries an image, prepend the + // recognition skill body so the model recognizes it before other actions. + // Delegate first, then prepend onto the decision (never veto, never rewrite). + ctx.on('agent/pre-step', async ( + { agent, signal }, + next, + ): Promise => { + const decision = await next() + if (decision.kind === 'reject') return decision + if (!hasImageSignal(decision.messages, detectBlocks, detectPaths)) return decision + signal.throwIfAborted() + const lookup = { cwd: agent.session.header.cwd, signal, scope: agent } + const skill = await ctx.skills.get(SKILL_NAME, lookup) + signal.throwIfAborted() + if (skill === undefined || !isModelInvocable(skill)) return decision + const injection = createUserMessage({ + content: [{ type: 'text', text: renderSkillContent(skill) }], + source: { ...PLUGIN_SOURCE, form: 'instructions', summary: 'recognize image first' }, + }) + return { kind: 'enter', messages: [injection, ...decision.messages] } + }) + + ctx.systemPrompt.section({ + name: 'tool:image-recognition', + order: 115, + text: 'When a task involves an image, recognize it first with `recognize_image` before continuing; do not infer image contents from a filename or description.', + }) +} diff --git a/packages/vision/tool-image-recognition/src/invariant.ts b/packages/vision/tool-image-recognition/src/invariant.ts new file mode 100644 index 0000000000..d0bf1cde5c --- /dev/null +++ b/packages/vision/tool-image-recognition/src/invariant.ts @@ -0,0 +1,31 @@ +/** + * Package-owned invariant companion for `@deepseek-ai/dsh-tool-image-recognition`. + * @module @deepseek-ai/dsh-tool-image-recognition/invariant + */ + +/* jscpd:ignore-start */ +import type { Context } from '@deepseek-ai/cordis' +import type { InvariantInstaller } from '@deepseek-ai/dsh-invariants' + +const PACKAGE_NAME = '@deepseek-ai/dsh-tool-image-recognition' + +/** Cordis companion plugin name. */ +export const name = 'tool-image-recognition-invariant' +/** Service required before the companion can reserve package ownership. */ +export const inject = ['invariants'] + +/** + * No runtime invariant: the consumer registers into the tools/skills registries + * and its injection is asserted by its own tests; it publishes no independent + * observation stream. + */ +const install: InvariantInstaller = () => {} + +/** + * Register this package's invariant companion. + * @param ctx - Cordis context carrying the invariant service. + * @returns the installed registration's disposer after setup succeeds. + */ +export const apply = (ctx: Context): Promise<() => void> => + Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install)) +/* jscpd:ignore-end */ diff --git a/packages/vision/tool-image-recognition/tests/has-image-signal.spec.ts b/packages/vision/tool-image-recognition/tests/has-image-signal.spec.ts new file mode 100644 index 0000000000..a00168a7d3 --- /dev/null +++ b/packages/vision/tool-image-recognition/tests/has-image-signal.spec.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest' +import type { UserMessage } from '@deepseek-ai/dsh-session' +import { hasImageSignal } from '../src/index.ts' + +function userMessage(blocks: UserMessage['content']): UserMessage { + return { id: 'u', role: 'user', source: { kind: 'user' }, content: blocks } as UserMessage +} + +/** A content block carrying an image signal (the payload is irrelevant here). */ +function imageBlock(): UserMessage['content'][number] { + return { type: 'image', attachment: {} as never } as UserMessage['content'][number] +} + +describe('hasImageSignal', () => { + it('detects an image content block when block detection is on', () => { + expect(hasImageSignal([userMessage([imageBlock()])], true, true)).toBe(true) + }) + + it('ignores image blocks when block detection is off', () => { + expect(hasImageSignal([userMessage([imageBlock()])], false, true)).toBe(false) + }) + + it('detects an image file path in text when path detection is on', () => { + const message = userMessage([{ type: 'text', text: 'transcribe /tmp/screenshot.png' }]) + expect(hasImageSignal([message], true, true)).toBe(true) + }) + + it('detects an image URL in text', () => { + const message = userMessage([{ type: 'text', text: 'look at https://e.test/photo.JPG' }]) + expect(hasImageSignal([message], true, true)).toBe(true) + }) + + it('ignores plain text with no image signal', () => { + const message = userMessage([{ type: 'text', text: 'summarize this document' }]) + expect(hasImageSignal([message], true, true)).toBe(false) + }) +}) diff --git a/packages/vision/tool-image-recognition/tsconfig.json b/packages/vision/tool-image-recognition/tsconfig.json new file mode 100644 index 0000000000..a45e88317f --- /dev/null +++ b/packages/vision/tool-image-recognition/tsconfig.json @@ -0,0 +1,39 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../image-recognition" + }, + { + "path": "../../core/agent" + }, + { + "path": "../../core/session" + }, + { + "path": "../../llm/llm" + }, + { + "path": "../../core/tools" + }, + { + "path": "../../skill/skill" + }, + { + "path": "../../runtime-diagnostics/invariants" + } + ] +}