diff --git a/packages/vision/image-recognition/package.json b/packages/vision/image-recognition/package.json new file mode 100644 index 0000000000..601d4ce19c --- /dev/null +++ b/packages/vision/image-recognition/package.json @@ -0,0 +1,47 @@ +{ + "name": "@deepseek-ai/dsh-image-recognition", + "description": "Abstract image-recognition capability seam (ctx.imageRecognition) for the DeepSeek Harness — provider registry, registration-order-independent selection, request/result vocabulary, and the ImageRecognitionError taxonomy", + "version": "0.1.0-rc.5", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", + "directory": "packages/vision/image-recognition" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./invariant": { + "types": "./lib/types/invariant.d.ts", + "default": "./lib/invariant.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/invariant.js", + "lib/types/**/*.d.ts" + ], + "license": "MIT", + "peerDependencies": { + "@deepseek-ai/dsh-invariants": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/cordis": "workspace:^" + }, + "dependencies": { + "@deepseek-ai/schemastery": "workspace:^" + }, + "devDependencies": { + "@deepseek-ai/dsh-invariants": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/cordis": "workspace:^" + } +} diff --git a/packages/vision/image-recognition/src/index.ts b/packages/vision/image-recognition/src/index.ts new file mode 100644 index 0000000000..17c269578d --- /dev/null +++ b/packages/vision/image-recognition/src/index.ts @@ -0,0 +1,156 @@ +/** + * Service Definition for the image-recognition capability seam + * (`ctx.imageRecognition`): a provider registry and provider-selecting + * execution. Duplicate ids are rejected. At execution time, a configured + * provider must exist and be usable; without one, exactly one usable provider is + * required, so selection never depends on registration order. + * @module @deepseek-ai/dsh-image-recognition + */ + +import { Context, Service } from '@deepseek-ai/cordis' +import z from '@deepseek-ai/schemastery' +import { + ImageRecognitionError, + type ImageRecognitionProvider, + type ImageRecognitionRequest, + type ImageRecognitionResult, +} from './types.ts' + +export { ImageRecognitionError } from './types.ts' +export type { + ImageInput, + ImageRecognitionProvider, + ImageRecognitionRequest, + ImageRecognitionResult, +} from './types.ts' + +declare module '@deepseek-ai/cordis' { + interface Context { + imageRecognition: ImageRecognitionRuntime + } +} + +/** Selection inputs for execution-time provider resolution. */ +interface Selection { + /** The configured provider id for this capability, if any. */ + readonly configuredId?: string + /** Providers registered for this capability kind. */ + readonly providers: ReadonlyMap +} + +/** + * Config for the image-recognition seam. `provider` pins which provider wins; + * optional (a single registered usable provider auto-selects). Operational + * overrides such as environment variables feed these same fields rather than + * introduce a hidden priority chain. + */ +export interface ImageRecognitionRuntimeConfig { + /** Explicit provider id. Omitted = auto-select when exactly one usable. */ + readonly provider?: string +} + +/** + * The image-recognition service. Registered as `ctx.imageRecognition` (one + * instance per context). + * + * Selection semantics (resolved at execution time, never order-dependent): + * - A configured id registered and `available()` → that provider. + * - A configured id not registered → `IMAGE_RECOGNITION_PROVIDER_CONFIGURED_MISSING`. + * - A configured id registered but unavailable → `IMAGE_RECOGNITION_PROVIDER_CONFIGURED_UNAVAILABLE`. + * - No id, exactly one registered usable provider → that provider. + * - No id, multiple usable providers → `IMAGE_RECOGNITION_PROVIDER_AMBIGUOUS`. + * - No id, no usable provider → `IMAGE_RECOGNITION_PROVIDER_UNAVAILABLE`. + */ +export class ImageRecognitionRuntime extends Service { + /** + * Provider selection config. `$DSH_IMAGE_RECOGNITION_PROVIDER` is equivalent + * to `provider` and is NOT a hidden priority chain. + */ + static Config: z = z.object({ + provider: z.string(), + }) + + private providers = new Map() + private readonly providerId: string | undefined + + constructor(ctx: Context, config: ImageRecognitionRuntimeConfig = {}) { + super(ctx, 'imageRecognition') + this.providerId = config.provider ?? process.env.DSH_IMAGE_RECOGNITION_PROVIDER + } + + /** + * Register a recognition provider. Throws {@link ImageRecognitionError} + * `IMAGE_RECOGNITION_DUPLICATE_PROVIDER` if its id is already registered. + * Returns a disposer; disposed with the calling fiber. + * @param provider - the provider; its `id` is the registry key. + * @returns the disposer that unregisters the provider. + */ + registerProvider(provider: ImageRecognitionProvider): () => void { + const providers = this.providers + if (providers.has(provider.id)) { + throw new ImageRecognitionError( + `an image-recognition provider with id "${provider.id}" is already registered`, + 'IMAGE_RECOGNITION_DUPLICATE_PROVIDER', + ) + } + const dispose = this.ctx.effect(function* () { + providers.set(provider.id, provider) + yield () => providers.delete(provider.id) + }, 'imageRecognition.registerProvider()') + // ctx.effect's disposer returns Promise; our disposer API is + // synchronous fire-and-forget — discard the (always-resolved) promise. + return () => void dispose() + } + + /** + * Run one recognition through the selected provider. Resolves the provider at + * call time with the selection rules above; throws {@link ImageRecognitionError} + * when the capability cannot run. + * @param request - the image and optional recognition prompt. + * @param signal - optional cancellation signal forwarded to the provider. + * @returns the recognized text. + */ + async recognize(request: ImageRecognitionRequest, signal?: AbortSignal): Promise { + const provider = resolveProvider({ + providers: this.providers, + ...this.providerId !== undefined ? { configuredId: this.providerId } : {}, + }) + return provider.recognize(request, signal) + } +} + +/** Resolve the selected provider or throw the matching {@link ImageRecognitionError}. */ +function resolveProvider(selection: Selection): ImageRecognitionProvider { + const { configuredId, providers } = selection + if (configuredId !== undefined) { + const provider = providers.get(configuredId) + if (!provider) { + throw new ImageRecognitionError( + `configured image-recognition provider "${configuredId}" is not registered`, + 'IMAGE_RECOGNITION_PROVIDER_CONFIGURED_MISSING', + ) + } + if (!provider.available()) { + throw new ImageRecognitionError( + `configured image-recognition provider "${configuredId}" is registered but unavailable`, + 'IMAGE_RECOGNITION_PROVIDER_CONFIGURED_UNAVAILABLE', + ) + } + return provider + } + const usable = [...providers.values()].filter(provider => provider.available()) + const [single] = usable + if (single === undefined) { + throw new ImageRecognitionError('no usable image-recognition provider is registered', 'IMAGE_RECOGNITION_PROVIDER_UNAVAILABLE') + } + if (usable.length > 1) { + const ids = usable.map(provider => provider.id).join(', ') + throw new ImageRecognitionError( + `multiple usable image-recognition providers are registered (${ids}); configure one explicitly`, + 'IMAGE_RECOGNITION_PROVIDER_AMBIGUOUS', + ) + } + return single +} + +export default ImageRecognitionRuntime diff --git a/packages/vision/image-recognition/src/invariant.ts b/packages/vision/image-recognition/src/invariant.ts new file mode 100644 index 0000000000..20589248a3 --- /dev/null +++ b/packages/vision/image-recognition/src/invariant.ts @@ -0,0 +1,30 @@ +/** + * Package-owned invariant companion for `@deepseek-ai/dsh-image-recognition`. + * @module @deepseek-ai/dsh-image-recognition/invariant + */ + +/* jscpd:ignore-start */ +import type { Context } from '@deepseek-ai/cordis' +import type { InvariantInstaller } from '@deepseek-ai/dsh-invariants' + +const PACKAGE_NAME = '@deepseek-ai/dsh-image-recognition' + +/** Cordis companion plugin name. */ +export const name = 'image-recognition-invariant' +/** Service required before the companion can reserve package ownership. */ +export const inject = ['invariants'] + +/** + * No runtime invariant: the provider map is private and selection is enforced on + * each call; the seam publishes no independent registry or observation stream. + */ +const install: InvariantInstaller = () => {} + +/** + * Register this package's invariant companion. + * @param ctx - Cordis context carrying the invariant service. + * @returns the installed registration's disposer after setup succeeds. + */ +export const apply = (ctx: Context): Promise<() => void> => + Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install)) +/* jscpd:ignore-end */ diff --git a/packages/vision/image-recognition/src/types.ts b/packages/vision/image-recognition/src/types.ts new file mode 100644 index 0000000000..2b7cba423c --- /dev/null +++ b/packages/vision/image-recognition/src/types.ts @@ -0,0 +1,60 @@ +/** + * Vocabulary for the image-recognition capability seam (`ctx.imageRecognition`). + * A provider recognizes image content through a user-configured endpoint and + * returns text the model can continue a task with. The request carries the image + * to inspect; the result is recognized text. Mirrors the web seam so provider + * selection, cancellation, errors, and configuration share one owner. + * @module @deepseek-ai/dsh-image-recognition/types + */ + +import { HarnessError } from '@deepseek-ai/dsh-llm' + +/** + * The image a recognition provider should inspect. A CLOSED discriminated union: + * the provider encodes the kind; consumers pass the source they hold. A new kind + * is a coordinated change across the seam, not a plugin extension. + */ +export type ImageInput = + | { readonly kind: 'file-path'; readonly filePath: string } + | { readonly kind: 'base64'; readonly base64: string; readonly mediaType: string } + | { readonly kind: 'url'; readonly url: string } + +/** + * What one recognition-capable backend is asked to do. `prompt` is an optional + * instruction for what to recognize (e.g. "transcribe the text"); omitted, the + * provider returns a general description. Cancellation is a direct execution + * argument, not a request field. + */ +export interface ImageRecognitionRequest { + /** The image to recognize. */ + readonly image: ImageInput + /** Optional recognition focus; absent = general description. */ + readonly prompt?: string +} + +/** Normalized recognition outcome: the recognized text. */ +export interface ImageRecognitionResult { + /** Recognized or described content the model can act on. */ + readonly text: string +} + +/** + * A recognition-capable backend. Registered with `ctx.imageRecognition`. + * `id` is a stable string, unique within the capability kind. + */ +export interface ImageRecognitionProvider { + /** Stable unique id used as the registry key and selection pin. */ + readonly id: string + /** Cheap local usability check; must not make network calls. */ + available(): boolean + /** Recognize one image; honor `signal` for cancellation. */ + recognize(request: ImageRecognitionRequest, signal?: AbortSignal): Promise +} + +/** + * Typed image-recognition error with a machine-routable, open-string `code` and + * chained `cause`. Consumers must tolerate provider-specific codes. Shared codes + * cover unavailable, missing, unusable, ambiguous, or duplicate providers, + * cancellation, and provider failure. + */ +export class ImageRecognitionError extends HarnessError {} diff --git a/packages/vision/image-recognition/tests/image-recognition.spec.ts b/packages/vision/image-recognition/tests/image-recognition.spec.ts new file mode 100644 index 0000000000..a97cb6f5b6 --- /dev/null +++ b/packages/vision/image-recognition/tests/image-recognition.spec.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' +import { Context } from '@deepseek-ai/cordis' +import ImageRecognitionRuntime, { + type ImageRecognitionProvider, + type ImageRecognitionRequest, + type ImageRecognitionResult, +} from '@deepseek-ai/dsh-image-recognition' + +/** A scripted recognition provider for contract tests. */ +function makeProvider( + id: string, + available: boolean, + recognize: (request: ImageRecognitionRequest) => Promise, +): ImageRecognitionProvider { + return { id, available: () => available, recognize: request => recognize(request) } +} + +const available = true +const unavailable = false + +function recognizeResult(marker: string): ImageRecognitionResult { + return { text: marker } +} + +/** Mount an ImageRecognitionRuntime on a fresh root context with the given config. */ +async function mountRuntime( + config: ConstructorParameters[1] = {}, +): Promise<{ ctx: Context; runtime: ImageRecognitionRuntime }> { + const ctx = new Context() + await ctx.plugin(ImageRecognitionRuntime, config) + return { ctx, runtime: ctx.imageRecognition } +} + +describe('ImageRecognitionRuntime registration', () => { + it('registers a provider and unregisters it via the returned disposer', async () => { + const { runtime } = await mountRuntime() + + const dispose = runtime.registerProvider(makeProvider('http', available, () => Promise.resolve(recognizeResult('cat')))) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })).resolves.toMatchObject({ text: 'cat' }) + + dispose() + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })) + .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_UNAVAILABLE' })) + }) + + it('throws IMAGE_RECOGNITION_DUPLICATE_PROVIDER on a duplicate id', async () => { + const { runtime } = await mountRuntime() + runtime.registerProvider(makeProvider('http', available, () => Promise.resolve(recognizeResult('a')))) + expect(() => runtime.registerProvider(makeProvider('http', available, () => Promise.resolve(recognizeResult('a'))))) + .toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_DUPLICATE_PROVIDER' })) + }) + + it('disposes provider registrations when the contributing fiber is disposed (HMR safety)', async () => { + const { ctx, runtime } = await mountRuntime() + const fiber = await ctx.plugin(Object.assign((inner: Context) => { + inner.imageRecognition.registerProvider(makeProvider('http', available, () => Promise.resolve(recognizeResult('a')))) + }, { inject: ['imageRecognition'] })) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })).resolves.toMatchObject({ text: 'a' }) + await fiber.dispose() + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })) + .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_UNAVAILABLE' })) + }) +}) + +describe('ImageRecognitionRuntime selection', () => { + it('picks the configured provider when registered and available', async () => { + const { runtime } = await mountRuntime({ provider: 'b' }) + runtime.registerProvider(makeProvider('a', available, () => Promise.resolve(recognizeResult('a')))) + runtime.registerProvider(makeProvider('b', available, () => Promise.resolve(recognizeResult('b')))) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })).resolves.toMatchObject({ text: 'b' }) + }) + + it('rejects a configured provider that is not registered', async () => { + const { runtime } = await mountRuntime({ provider: 'missing' }) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })) + .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_CONFIGURED_MISSING' })) + }) + + it('rejects a configured provider that is registered but unavailable', async () => { + const { runtime } = await mountRuntime({ provider: 'http' }) + runtime.registerProvider(makeProvider('http', unavailable, () => Promise.resolve(recognizeResult('a')))) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })) + .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_CONFIGURED_UNAVAILABLE' })) + }) + + it('auto-selects the single usable provider when none is configured', async () => { + const { runtime } = await mountRuntime() + runtime.registerProvider(makeProvider('http', unavailable, () => Promise.resolve(recognizeResult('a')))) + runtime.registerProvider(makeProvider('other', available, () => Promise.resolve(recognizeResult('other')))) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })).resolves.toMatchObject({ text: 'other' }) + }) + + it('rejects multiple usable providers without a configured pin', async () => { + const { runtime } = await mountRuntime() + runtime.registerProvider(makeProvider('a', available, () => Promise.resolve(recognizeResult('a')))) + runtime.registerProvider(makeProvider('b', available, () => Promise.resolve(recognizeResult('b')))) + await expect(runtime.recognize({ image: { kind: 'url', url: 'https://e.test/c.png' } })) + .rejects.toThrow(expect.objectContaining({ code: 'IMAGE_RECOGNITION_PROVIDER_AMBIGUOUS' })) + }) +}) diff --git a/packages/vision/image-recognition/tsconfig.json b/packages/vision/image-recognition/tsconfig.json new file mode 100644 index 0000000000..58798daeb8 --- /dev/null +++ b/packages/vision/image-recognition/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../../llm/llm" + }, + { + "path": "../../runtime-diagnostics/invariants" + } + ] +}