Add web capability seam: ctx.web, search/fetch providers, web tools

Introduce web access as a first-class capability seam so the model-facing web tools stay stable while backends change. dsh-web owns ctx.web as a provider registry with registration-order-independent selection and the WebError taxonomy; dsh-web-search-exa, dsh-web-search-perplexity, and dsh-web-fetch-local register capabilities into it; dsh-tool-web is the sole owner of the model-facing web_search/web_fetch schemas, prompt sections, and HTML-to-markdown presentation. Search and fetch are deliberately one seam. Providers ship as namespace plugins that register into ctx.web (like an LlmAdapter into ctx.llm), not key-owning services, since multiple search providers cannot each own the key. Tool registration follows product enablement, not backend availability, so load order/credentials never enter the model contract; the seam resolves the provider at execution time and surfaces a structured WebError otherwise. Moves the RFC to implemented/ amended to match what shipped. Example/app configs are intentionally not wired yet (RFC migration step 6).
2026-06-25 15:04:12 +08:00
parent a4091daa3d
commit d01f5f73b7
50 changed files with 3610 additions and 8 deletions
@@ -0,0 +1,77 @@
+/**
+ * `@deepseek-ai/dsh-web-fetch-local`: registers an anonymous public HTTP(S)
+ * `WebFetchProvider` with `ctx.web`. A function/namespace plugin (NOT a
+ * default-export service): it registers INTO the seam's fetch registry, like the
+ * search providers register into the search registry.
+ *
+ * @module @deepseek-ai/dsh-web-fetch-local
+ */
+
+import type { Context } from 'cordis'
+import z from 'schemastery'
+import type {} from '@deepseek-ai/dsh-web'
+import { LocalFetchProvider } from './provider.ts'
+import type { LocalFetchLimits } from './provider.ts'
+
+export {
+  LOCAL_FETCH_PROVIDER_ID,
+  LocalFetchProvider,
+} from './provider.ts'
+export type { LocalFetchLimits } from './provider.ts'
+export { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts'
+export type { FetchableKind } from './policy.ts'
+
+/** Default `User-Agent`: an explicit product agent, never a browser disguise. */
+export const DEFAULT_USER_AGENT = 'deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)'
+
+/** Cordis plugin name used by loader diagnostics. */
+export const name = 'web-fetch-local'
+
+/** The web seam this provider registers into. */
+export const inject = ['web']
+
+export interface Config {
+  /** Maximum accepted request URL length. */
+  maxUrlLength?: number
+  /** Maximum response body size in bytes. */
+  maxResponseBytes?: number
+  /** Maximum decoded body length in characters. */
+  maxBodyChars?: number
+  /** Default fetch timeout in milliseconds. */
+  timeoutMs?: number
+  /** Upper bound for a per-request timeout override. */
+  maxTimeoutMs?: number
+  /** Maximum number of same-origin redirect hops to follow. */
+  maxRedirects?: number
+  /** `User-Agent` header sent on every request. */
+  userAgent?: string
+}
+
+export const Config: z<Config> = z.object({
+  maxUrlLength: z.number().default(2048),
+  maxResponseBytes: z.number().default(5_000_000),
+  maxBodyChars: z.number().default(100_000),
+  timeoutMs: z.number().default(30_000),
+  maxTimeoutMs: z.number().default(120_000),
+  maxRedirects: z.number().default(5),
+  userAgent: z.string().default(DEFAULT_USER_AGENT),
+})
+
+/** The shape after schemastery applies its defaults to every field. */
+type ResolvedConfig = Required<Config>
+
+/** Register the local HTTP(S) fetch provider with `ctx.web`. */
+export function apply(ctx: Context, config: Config): void {
+  // schemastery (Config) has already filled every defaulted field.
+  const resolved = config as ResolvedConfig
+  const limits: LocalFetchLimits = {
+    maxUrlLength: resolved.maxUrlLength,
+    maxResponseBytes: resolved.maxResponseBytes,
+    maxBodyChars: resolved.maxBodyChars,
+    timeoutMs: resolved.timeoutMs,
+    maxTimeoutMs: resolved.maxTimeoutMs,
+    maxRedirects: resolved.maxRedirects,
+    userAgent: resolved.userAgent,
+  }
+  ctx.web.registerFetchProvider(new LocalFetchProvider(limits))
+}
@@ -0,0 +1,59 @@
+/**
+ * URL validation and content-type classification for the local HTTP(S) fetch
+ * provider — the pure, network-free half. The provider's `fetch()` composes
+ * these with transport (redirect following, byte caps, decoding).
+ *
+ * @module @deepseek-ai/dsh-web-fetch-local/policy
+ */
+
+import { WebError } from '@deepseek-ai/dsh-web'
+
+/** The body kinds this provider decodes. */
+export type FetchableKind = 'html' | 'text'
+
+/**
+ * Validate a request URL against the basic transport hygiene the provider
+ * enforces before any network access: http(s) only, no embedded credentials,
+ * bounded length. Returns the parsed `URL`. Throws {@link WebError} otherwise.
+ * (SSRF / private-network blocking is deferred — see the package RFC.)
+ */
+export function validateFetchUrl(input: string, maxUrlLength: number): URL {
+  if (input.length > maxUrlLength) {
+    throw new WebError(`URL exceeds the maximum length of ${maxUrlLength}`, 'WEB_INVALID_URL')
+  }
+  let url: URL
+  try {
+    url = new URL(input)
+  } catch (error: unknown) {
+    throw new WebError(`invalid URL: ${input}`, 'WEB_INVALID_URL', { cause: error })
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+    throw new WebError(`unsupported URL scheme "${url.protocol}" (only http and https are allowed)`, 'WEB_INVALID_URL')
+  }
+  if (url.username.length > 0 || url.password.length > 0) {
+    throw new WebError('credentials in URLs are not allowed', 'WEB_BLOCKED_URL')
+  }
+  return url
+}
+
+/**
+ * Two URLs are same-origin when scheme, hostname, and port match. A redirect
+ * that crosses origins is refused so each new origin requires a fresh tool call
+ * (and thus a fresh provider/permission decision).
+ */
+export function isSameOrigin(a: URL, b: URL): boolean {
+  return a.protocol === b.protocol && a.hostname === b.hostname && a.port === b.port
+}
+
+/**
+ * Classify a response `Content-Type` into a decodable body kind, or `undefined`
+ * for an unsupported (e.g. binary) type. `text/html` and `application/xhtml+xml`
+ * are `html`; other `text/*` plus a few structured text types are `text`.
+ */
+export function classifyContentType(contentType: string | null): FetchableKind | undefined {
+  const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
+  if (mime === 'text/html' || mime === 'application/xhtml+xml') return 'html'
+  if (mime.startsWith('text/')) return 'text'
+  if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
+  return undefined
+}
@@ -0,0 +1,233 @@
+/**
+ * `LocalFetchProvider`: a `WebFetchProvider` that retrieves a concrete public
+ * HTTP(S) URL with the platform-native `fetch` (Node 24) and returns a status
+ * code plus bounded decoded content. It owns SAFE RESOURCE RETRIEVAL — URL
+ * validation, redirect policy, timeout, abort, byte caps, charset decoding,
+ * content-type classification, binary rejection — but NOT presentation
+ * (HTML→markdown lives in `@deepseek-ai/dsh-tool-web`).
+ *
+ * Redirects are followed manually (`redirect: 'manual'`) so the provider can
+ * enforce a same-origin-only policy: a cross-origin redirect is refused with
+ * `WEB_REDIRECT_BLOCKED`, requiring a fresh tool call (Claude Code's WebFetch
+ * uses the same model). It does NOT carry browser cookies, editor/git
+ * credentials, or implicit access to private services.
+ *
+ * SSRF / private-network protection is DEFERRED (see the package RFC); until it
+ * lands this provider is an SSRF primitive and must not be enabled where it can
+ * reach sensitive internal targets.
+ *
+ * @module @deepseek-ai/dsh-web-fetch-local/provider
+ */
+
+import { WebError } from '@deepseek-ai/dsh-web'
+import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult, WebProviderStatus } from '@deepseek-ai/dsh-web'
+import { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts'
+
+/** Resolved provider limits (the plugin's schemastery Config supplies defaults). */
+export interface LocalFetchLimits {
+  /** Maximum accepted request URL length. */
+  maxUrlLength: number
+  /** Maximum response body size in bytes (read is aborted past this). */
+  maxResponseBytes: number
+  /** Maximum decoded body length in characters (truncated past this). */
+  maxBodyChars: number
+  /** Default fetch timeout in milliseconds. */
+  timeoutMs: number
+  /** Upper bound for a per-request timeout override. */
+  maxTimeoutMs: number
+  /** Maximum number of (same-origin) redirect hops to follow. */
+  maxRedirects: number
+  /** `User-Agent` header sent on every request. */
+  userAgent: string
+}
+
+/** Stable id this provider registers under. */
+export const LOCAL_FETCH_PROVIDER_ID = 'local-http'
+
+/** The anonymous public HTTP(S) fetch provider. */
+export class LocalFetchProvider implements WebFetchProvider {
+  readonly id = LOCAL_FETCH_PROVIDER_ID
+
+  constructor(private readonly limits: LocalFetchLimits) {}
+
+  /** No credentials to check — an anonymous public fetcher is always usable. */
+  status(): WebProviderStatus {
+    return { available: true }
+  }
+
+  async fetch(request: WebFetchRequest, exec?: { readonly signal?: AbortSignal }): Promise<WebFetchResult> {
+    const timeoutMs = request.timeoutMs !== undefined
+      ? Math.min(request.timeoutMs, this.limits.maxTimeoutMs)
+      : this.limits.timeoutMs
+
+    // One controller drives both the caller's abort and our own timeout, so the
+    // network request and the streaming read both stop on either.
+    const controller = new AbortController()
+    const onAbort = (): void => { controller.abort() }
+    if (exec?.signal !== undefined) {
+      if (exec.signal.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED')
+      exec.signal.addEventListener('abort', onAbort, { once: true })
+    }
+    const timer = setTimeout(() => { controller.abort(new WebError('web fetch timed out', 'WEB_FETCH_TIMEOUT')) }, timeoutMs)
+
+    try {
+      return await this.followAndRead(request.url, controller, timeoutMs)
+    } finally {
+      clearTimeout(timer)
+      if (exec?.signal !== undefined) exec.signal.removeEventListener('abort', onAbort)
+    }
+  }
+
+  /** Follow same-origin redirects up to the hop cap, then read the final response. */
+  private async followAndRead(initialUrl: string, controller: AbortController, timeoutMs: number): Promise<WebFetchResult> {
+    let currentUrl = validateFetchUrl(initialUrl, this.limits.maxUrlLength)
+
+    for (let hop = 0; hop <= this.limits.maxRedirects; hop++) {
+      const response = await this.requestOnce(currentUrl, controller, timeoutMs)
+
+      if (isRedirectStatus(response.status)) {
+        const location = response.headers.get('location')
+        if (location === null) {
+          // A redirect status with no Location is not a usable resource.
+          throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, 'WEB_PROVIDER_ERROR')
+        }
+        const target = resolveRedirect(location, currentUrl)
+        if (!isSameOrigin(target, currentUrl)) {
+          throw new WebError(
+            `cross-origin redirect to ${target.origin} is not followed automatically; retry against that URL directly`,
+            'WEB_REDIRECT_BLOCKED',
+          )
+        }
+        await response.body?.cancel()
+        currentUrl = target
+        continue
+      }
+
+      return await this.readBody(response, currentUrl)
+    }
+
+    throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, 'WEB_REDIRECT_BLOCKED')
+  }
+
+  private async requestOnce(url: URL, controller: AbortController, _timeoutMs: number): Promise<Response> {
+    try {
+      return await fetch(url, {
+        method: 'GET',
+        redirect: 'manual',
+        headers: { 'user-agent': this.limits.userAgent, 'accept': 'text/html,application/xhtml+xml,text/*;q=0.9,application/json;q=0.8' },
+        signal: controller.signal,
+      })
+    } catch (error: unknown) {
+      throw translateAbortOrNetwork(error)
+    }
+  }
+
+  /** Read, byte-cap, classify, and decode the final response body. */
+  private async readBody(response: Response, finalUrl: URL): Promise<WebFetchResult> {
+    const kind = classifyContentType(response.headers.get('content-type'))
+    if (kind === undefined) {
+      await response.body?.cancel()
+      throw new WebError(`unsupported content type "${response.headers.get('content-type') ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
+    }
+
+    const { bytes, truncatedByBytes } = await this.readCapped(response)
+    const decoded = new TextDecoder('utf-8').decode(bytes)
+    const truncatedByChars = decoded.length > this.limits.maxBodyChars
+    const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded
+    const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content }
+
+    return {
+      providerId: this.id,
+      url: finalUrl.toString(),
+      statusCode: response.status,
+      body,
+      truncated: truncatedByBytes || truncatedByChars,
+    }
+  }
+
+  /**
+   * Read the response stream up to `maxResponseBytes`. A `Content-Length` over
+   * the cap rejects immediately with `WEB_FETCH_TOO_LARGE`; a stream that grows
+   * past the cap is cut short (`truncatedByBytes`) rather than rejected, so a
+   * server that under-reports still yields a bounded usable body.
+   */
+  private async readCapped(response: Response): Promise<{ bytes: Uint8Array; truncatedByBytes: boolean }> {
+    const declared = response.headers.get('content-length')
+    if (declared !== null) {
+      const length = Number(declared)
+      if (Number.isFinite(length) && length > this.limits.maxResponseBytes) {
+        await response.body?.cancel()
+        throw new WebError(`response exceeds the maximum of ${this.limits.maxResponseBytes} bytes`, 'WEB_FETCH_TOO_LARGE')
+      }
+    }
+
+    /* v8 ignore next -- a 2xx Response from fetch always exposes a body stream; the null guard is defensive. */
+    if (response.body === null) return { bytes: new Uint8Array(0), truncatedByBytes: false }
+
+    const chunks: Uint8Array[] = []
+    let total = 0
+    let truncatedByBytes = false
+    const reader = response.body.getReader()
+    try {
+      for (;;) {
+        const { done, value } = await reader.read()
+        if (done) break
+        const remaining = this.limits.maxResponseBytes - total
+        if (value.byteLength >= remaining) {
+          chunks.push(value.subarray(0, remaining))
+          total += remaining
+          truncatedByBytes = true
+          break
+        }
+        chunks.push(value)
+        total += value.byteLength
+      }
+    } catch (error: unknown) {
+      /* v8 ignore next -- mid-stream read fault needs a network drop after headers; translate path covered by request-phase tests. */
+      throw translateAbortOrNetwork(error)
+    } finally {
+      /* v8 ignore next 4 -- cancel() after a completed/broken read settles without rejecting; unobserved best-effort cleanup. */
+      await reader.cancel().catch(() => {
+        // Cancel after a successful read (or after we broke past the cap) is
+        // best-effort cleanup; the bytes we need are already collected.
+      })
+    }
+
+    const bytes = new Uint8Array(total)
+    let offset = 0
+    for (const chunk of chunks) {
+      bytes.set(chunk, offset)
+      offset += chunk.byteLength
+    }
+    return { bytes, truncatedByBytes }
+  }
+}
+
+/** HTTP redirect status codes that carry a `Location`. */
+function isRedirectStatus(status: number): boolean {
+  return status === 301 || status === 302 || status === 303 || status === 307 || status === 308
+}
+
+/** Resolve a (possibly relative) `Location` against the current URL. */
+function resolveRedirect(location: string, base: URL): URL {
+  try {
+    return new URL(location, base)
+  } catch (error: unknown) {
+    /* v8 ignore next 2 -- URL resolution against a valid absolute base effectively never throws; defensive guard. */
+    throw new WebError(`invalid redirect Location "${location}"`, 'WEB_PROVIDER_ERROR', { cause: error })
+  }
+}
+
+/**
+ * Translate a thrown fetch/stream error into a `WebError`. Our own
+ * `WEB_FETCH_TIMEOUT` (passed to `controller.abort(reason)`) and any other
+ * already-typed `WebError` pass through; an `AbortError` becomes `WEB_ABORTED`;
+ * anything else is a transport/network failure (`WEB_PROVIDER_ERROR`).
+ */
+function translateAbortOrNetwork(error: unknown): WebError {
+  if (error instanceof WebError) return error
+  if (error instanceof DOMException && error.name === 'AbortError') {
+    return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
+  }
+  return new WebError(`web fetch failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
+}