fix(token-meter): bound projection state via logged shadow prices

The contextBreakdown and contextPressure units carried the full priced
surface, so each session's persisted projection checkpoint grew without
bound. A surface replacement is now priced by the shadow-price event
logged directly before it — compact/summary for compaction, the new
compact/prune from tool-result pruning (priced through the injected
token meter) — and the unit states shrink to a fixed handful of numbers.
Regenerate the persistence/cordis/module/config catalogs.

packages/llm/token-meter/src/breakdown-projection.ts

@@ -6,11 +6,11 @@
  */
 
 import { z } from 'zod'
-import { canonicalHeader, isSurfaceEvent } from '@deepseek-ai/dsh-session'
+import { canonicalHeader } from '@deepseek-ai/dsh-session'
 import type { ProjectionDefinition } from '@deepseek-ai/dsh-session-projection'
-import type { TokenSurfaceNode } from './types.ts'
 import { estimateSystemTokens, estimateToolsTokens } from './estimate.ts'
-import { foldSurfaceTokens } from './surface-fold.ts'
+import { foldSurfaceProjection } from './surface-projection.ts'
+import type { ShadowPriceClaim } from './surface-projection.ts'
 // Import for the `contextBreakdown` SessionProjectionMap key merge.
 import type {} from './projection.ts'
 
@@ -18,8 +18,8 @@ interface ContextBreakdownState {
   systemTokens: number
   toolsTokens: number
   messageTokens: number
-  /** Priced surface nodes (plain JSON for the persisted projection cache). */
-  surface: TokenSurfaceNode[]
+  /** Shadow price armed by the immediately preceding metering event. */
+  claim?: ShadowPriceClaim
 }
 
 const breakdownSchema = z.object({
@@ -32,31 +32,38 @@ const breakdownSchema = z.object({
  * Token-meter's context-composition projection unit.
  *
  * Envelope figures are last-wins per `request/header`; the message figure
- * rides {@link foldSurfaceTokens} — the same fold the measurement service
- * replays — so it equals `measure().surfaceTokens` at every event boundary and
- * compaction shrinks it the way it shrinks the next request.
+ * rides {@link foldSurfaceProjection} — the same O(1) fold the occupancy
+ * projection uses — so it equals `measure().surfaceTokens` at every event
+ * boundary and compaction shrinks it by its logged shadow price, the way it
+ * shrinks the next request. The state is a fixed handful of numbers, so the
+ * persisted checkpoint stays O(1) over the session's life.
  */
 export const contextBreakdownProjectionDefinition:
 ProjectionDefinition<'contextBreakdown', ContextBreakdownState> = {
   key: 'contextBreakdown',
   schema: breakdownSchema,
-  init: () => ({ systemTokens: 0, toolsTokens: 0, messageTokens: 0, surface: [] }),
+  init: () => ({ systemTokens: 0, toolsTokens: 0, messageTokens: 0 }),
   apply: (state, event) => {
+    const fold = foldSurfaceProjection(state.claim, event)
+    let systemTokens = state.systemTokens
+    let toolsTokens = state.toolsTokens
     if (event.type === 'request/header') {
       const header = canonicalHeader(event.data.header)
-      const systemTokens = estimateSystemTokens(header)
-      const toolsTokens = estimateToolsTokens(header)
-      if (systemTokens === state.systemTokens && toolsTokens === state.toolsTokens) return state
-      return { ...state, systemTokens, toolsTokens }
+      systemTokens = estimateSystemTokens(header)
+      toolsTokens = estimateToolsTokens(header)
     }
-    if (!isSurfaceEvent(event)) return state
-    const fold = foldSurfaceTokens(state.surface, event)
+    if (systemTokens === state.systemTokens
+      && toolsTokens === state.toolsTokens
+      && fold.deltaTokens === 0
+      && fold.claim === undefined
+      && state.claim === undefined) return state
     return {
-      ...state,
+      systemTokens,
+      toolsTokens,
       messageTokens: state.messageTokens + fold.deltaTokens,
-      surface: fold.nodes,
+      ...fold.claim === undefined ? {} : { claim: fold.claim },
     }
   },
   view: ({ systemTokens, toolsTokens, messageTokens }) => ({ systemTokens, toolsTokens, messageTokens }),
-  stateVersion: 1,
+  stateVersion: 2,
 }

packages/llm/token-meter/src/invariant.ts

@@ -20,9 +20,11 @@ export const inject = ['invariants']
  * three projections do expose observation streams, but their schemas fix the
  * JSON payloads; the usage folds replace same-step samples, so totals need not
  * be monotone when a final sample corrects an earlier chunk, and the
- * composition fold shares `surface-fold.ts` with the measurement service,
- * which makes its message figure equal `measure().surfaceTokens` by
- * construction rather than by a relation worth observing at runtime.
+ * composition fold prices through the same `estimate.ts` heuristic as the
+ * measurement service and subtracts producer-logged shadow prices derived
+ * from that service's own nodes, which makes its message figure equal
+ * `measure().surfaceTokens` by construction rather than by a relation worth
+ * observing at runtime.
  */
 const install: InvariantInstaller = () => {}
 

packages/llm/token-meter/src/surface-fold.ts

@@ -1,10 +1,11 @@
 /**
- * The one positional surface fold, shared by the measurement service's replay
- * state and the pure `contextBreakdown` projection. Both answer "what does the
- * current model-visible conversation cost", so they MUST price and place every
- * node identically: a private copy in either owner would let the panel's
- * message figure drift away from `measure().surfaceTokens` with both sides
- * still passing their own tests.
+ * The measurement service's positional surface fold: the per-node priced
+ * surface `measure()` serves and compaction plans against. The projection
+ * units deliberately do NOT share this fold — their state must stay O(1)
+ * for the persisted checkpoint, so they ride `surface-projection.ts`'s
+ * shadow-price protocol instead. The two stay in agreement by construction:
+ * both price through `estimate.ts`, and every logged shadow price is derived
+ * from THIS fold's nodes by the replace producer.
  *
  * @module @deepseek-ai/dsh-token-meter/surface-fold
  */

packages/llm/token-meter/src/surface-projection.ts

@@ -0,0 +1,84 @@
+/**
+ * The O(1) surface-token fold shared by the token-meter projection units.
+ *
+ * A projection state must stay bounded — the persisted projection cache
+ * checkpoints every unit's whole state, so carrying the priced surface
+ * (one node per model-visible message) would grow a checkpoint without
+ * bound over the session's life. Instead, replacements ride the compact
+ * seam's shadow-price protocol: the metering event immediately before a
+ * surface `replace` (`compact/summary` or `compact/prune`) states the
+ * heuristic price of the exact replaced range, so the fold keeps a running
+ * total plus at most one pending claim and never retains per-node prices.
+ * The counts are exact by construction: producers derive them from the same
+ * fixed estimator this module prices appends with.
+ *
+ * @module @deepseek-ai/dsh-token-meter/surface-projection
+ */
+
+import { deriveEventMessage, isSurfaceEvent } from '@deepseek-ai/dsh-session'
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+// Type-only: the `compact/*` SessionEventMap merges (shadow-price events).
+import type {} from '@deepseek-ai/dsh-compact'
+import { estimateMessage } from './estimate.ts'
+
+/**
+ * One armed shadow price: the heuristic tokens of the surface range the
+ * IMMEDIATELY following event replaces. Plain JSON — it is part of the
+ * persisted unit state while armed.
+ */
+export interface ShadowPriceClaim {
+  /** Declared inclusive first surface-node seq of the priced range. */
+  start: number
+  /** Declared inclusive last surface-node seq of the priced range. */
+  end: number
+  /** Heuristic tokens of the priced range under the fixed estimator. */
+  tokens: number
+}
+
+/** One event's effect on a running surface-token total. */
+export interface SurfaceTokensFold {
+  /** Signed change in the surface total; 0 for events off the surface. */
+  readonly deltaTokens: number
+  /** Claim to carry into the next event; undefined when none survives. */
+  readonly claim: ShadowPriceClaim | undefined
+}
+
+/**
+ * Fold one committed event onto a running surface-token total.
+ *
+ * A shadow-price event arms a claim; any other event expires it, and a
+ * surface `replace` must consume a claim naming its exact range — the
+ * producers append the metering event and the replacement synchronously
+ * adjacent, so a surviving claim always prices the very next event.
+ * @param claim - the claim armed by the immediately preceding event, if any.
+ * @param event - the next committed session event.
+ * @returns the signed token delta and the claim state after this event.
+ * @throws when a replacement arrives without a claim for its exact range —
+ *   every in-repo replace producer meters its replacement, so an unpriced
+ *   replacement is a shadow-price contract violation and must fail loud
+ *   rather than let the total drift.
+ */
+export function foldSurfaceProjection(
+  claim: ShadowPriceClaim | undefined,
+  event: SessionEvent,
+): SurfaceTokensFold {
+  if (event.type === 'compact/summary' || event.type === 'compact/prune') {
+    const { shadowedRange, shadowedTokenCount } = event.data
+    return {
+      deltaTokens: 0,
+      claim: { start: shadowedRange.start, end: shadowedRange.end, tokens: shadowedTokenCount },
+    }
+  }
+  if (!isSurfaceEvent(event)) return { deltaTokens: 0, claim: undefined }
+  const message = deriveEventMessage(event)
+  const tokens = message === null ? 0 : estimateMessage(message)
+  const op = event.surfaceOp
+  if (op === 'append') return { deltaTokens: tokens, claim: undefined }
+  if (claim === undefined || claim.start !== op.start || claim.end !== op.end) {
+    throw new Error(
+      `token surface: replace at seq ${event.seq} over range ${op.start}-${op.end} has no adjacent shadow price`
+      + (claim === undefined ? '' : ` (armed claim covers ${claim.start}-${claim.end})`),
+    )
+  }
+  return { deltaTokens: tokens - claim.tokens, claim: undefined }
+}

packages/llm/token-meter/src/usage-projection.ts

@@ -4,12 +4,11 @@
 
 import { z } from 'zod'
 import type { TokenUsage } from '@deepseek-ai/dsh-llm'
-import { isSurfaceEvent } from '@deepseek-ai/dsh-session'
 import type { SessionEvent } from '@deepseek-ai/dsh-session'
 import type { ProjectionDefinition } from '@deepseek-ai/dsh-session-projection'
 import type { ContextPressureProjection, TokenUsageProjection } from './projection.ts'
-import type { TokenSurfaceNode } from './types.ts'
-import { foldSurfaceTokens } from './surface-fold.ts'
+import { foldSurfaceProjection } from './surface-projection.ts'
+import type { ShadowPriceClaim } from './surface-projection.ts'
 
 interface UsageSample {
   turn: number
@@ -82,19 +81,25 @@ const usageOf = (event: SessionEvent): TokenUsage | undefined =>
 
 /**
  * Context-occupancy state: the two independent last-wins records plus the
- * priced surface needed to carry the newest sample forward.
+ * O(1) running surface total needed to carry the newest sample forward.
  */
 interface ContextPressureState {
   contextWindow?: number
   pressureTokens?: number
-  /** Priced surface, folded identically to the measurement service's. */
-  surface: TokenSurfaceNode[]
-  /** Summed heuristic tokens over {@link surface}. */
+  /** Running heuristic total over the current surface ({@link foldSurfaceProjection}). */
   surfaceTokens: number
   /** {@link surfaceTokens} at the newest usage sample; absent until one lands. */
   sampledSurfaceTokens?: number
+  /** Shadow price armed by the immediately preceding metering event. */
+  claim?: ShadowPriceClaim
 }
 
+/** Whether two optional shadow-price claims price the same range identically. */
+const claimEquals = (left: ShadowPriceClaim | undefined, right: ShadowPriceClaim | undefined): boolean =>
+  left === right
+  || (left !== undefined && right !== undefined
+    && left.start === right.start && left.end === right.end && left.tokens === right.tokens)
+
 /**
  * Token-meter's session projection unit.
  *
@@ -152,26 +157,33 @@ ProjectionDefinition<'tokenUsage', TokenUsageState> = {
  * `pressureTokens` is prompt-side only, so it holds still while a turn streams
  * and steps forward once the next request reports its usage. Because nothing
  * but a request reports usage, it also cannot see a compaction: the fold
- * therefore carries the priced surface alongside it and publishes
+ * therefore carries a running surface total alongside it and publishes
  * `projectedTokens` — the sample plus the surface's signed movement since it
  * was taken — so occupancy answers for the next request rather than the last
- * one. A usage sample is stamped BEFORE the same event joins the surface, so
- * an `assistant/message` anchors against the surface its own request saw.
+ * one. The total rides {@link foldSurfaceProjection}, so the state stays O(1)
+ * and a replacement shrinks it by its logged shadow price. A usage sample is
+ * stamped BEFORE the same event joins the surface, so an `assistant/message`
+ * anchors against the surface its own request saw.
  */
 export const contextPressureProjectionDefinition:
 ProjectionDefinition<'contextPressure', ContextPressureState> = {
   key: 'contextPressure',
   schema: pressureSchema,
-  init: () => ({ surface: [], surfaceTokens: 0 }),
+  init: () => ({ surfaceTokens: 0 }),
   apply: (state, event) => {
+    const fold = foldSurfaceProjection(state.claim, event)
+    let next = state
     if (event.type === 'request/context') {
       const contextWindow = event.data.contextWindow
-      if (contextWindow === state.contextWindow) return state
-      if (contextWindow !== undefined) return { ...state, contextWindow }
-      const { contextWindow: _removed, ...withoutContextWindow } = state
-      return withoutContextWindow
+      if (contextWindow !== state.contextWindow) {
+        if (contextWindow !== undefined) {
+          next = { ...next, contextWindow }
+        } else {
+          const { contextWindow: _removed, ...withoutContextWindow } = next
+          next = withoutContextWindow
+        }
+      }
     }
-    let next = state
     const usage = usageOf(event)
     if (usage !== undefined) {
       const pressureTokens = pressureFrom(usage)
@@ -179,9 +191,12 @@ ProjectionDefinition<'contextPressure', ContextPressureState> = {
         next = { ...next, pressureTokens, sampledSurfaceTokens: next.surfaceTokens }
       }
     }
-    if (!isSurfaceEvent(event)) return next
-    const fold = foldSurfaceTokens(next.surface, event)
-    return { ...next, surface: fold.nodes, surfaceTokens: next.surfaceTokens + fold.deltaTokens }
+    if (fold.deltaTokens !== 0) {
+      next = { ...next, surfaceTokens: next.surfaceTokens + fold.deltaTokens }
+    }
+    if (claimEquals(state.claim, fold.claim)) return next
+    const { claim: _expired, ...withoutClaim } = next
+    return fold.claim === undefined ? withoutClaim : { ...withoutClaim, claim: fold.claim }
   },
   view: ({ contextWindow, pressureTokens, surfaceTokens, sampledSurfaceTokens }) => ({
     ...contextWindow === undefined ? {} : { contextWindow },
@@ -190,5 +205,5 @@ ProjectionDefinition<'contextPressure', ContextPressureState> = {
       ? {}
       : { projectedTokens: Math.max(0, pressureTokens + surfaceTokens - sampledSurfaceTokens) },
   }),
-  stateVersion: 3,
+  stateVersion: 4,
 }