feat(llm-replay): resolve {{fromRequest:...}} placeholders against the live request

A scripted sidecar cannot know values minted at run time, so terminal
goal updates (which must echo the random goal id) were previously
un-scriptable. Placeholders in scripted entries now resolve against the
request corpus at stream time: last match wins, capture group 1 or the
whole match substitutes, and unmatched/invalid/unterminated patterns
fail loud.

packages/support/llm-replay/src/index.ts

@@ -241,6 +241,90 @@ const REPLAY_CHUNK_TYPES = new Set<StreamChunk['type']>([
   'finish',
 ])
 
+const FROM_REQUEST_OPEN = '{{fromRequest:'
+const FROM_REQUEST_CLOSE = '}}'
+
+/** Collect every string leaf of one JSON-shaped value, in traversal order. */
+function collectStrings(value: unknown, out: string[]): void {
+  if (typeof value === 'string') {
+    out.push(value)
+    return
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) collectStrings(item, out)
+    return
+  }
+  if (value !== null && typeof value === 'object') {
+    for (const item of Object.values(value)) collectStrings(item, out)
+  }
+}
+
+/** Resolve one placeholder pattern against the request corpus; the LAST match wins. */
+function resolveFromRequest(pattern: string, corpus: string): string {
+  let regex: RegExp
+  try {
+    regex = new RegExp(pattern, 'g')
+  } catch (error) {
+    // RegExp construction only throws SyntaxError; String() carries its message.
+    throw new Error(`llm-replay: fromRequest has an invalid pattern ${JSON.stringify(pattern)}: ${String(error)}`)
+  }
+  let last: RegExpExecArray | undefined
+  for (const match of corpus.matchAll(regex)) last = match
+  if (last === undefined) {
+    throw new Error(`llm-replay: fromRequest pattern ${JSON.stringify(pattern)} matched nothing in the request`)
+  }
+  return last[1] ?? last[0]
+}
+
+/** Replace every `{{fromRequest:<pattern>}}` occurrence in one scripted string. */
+function substituteString(text: string, corpus: string): string {
+  let result = ''
+  let cursor = 0
+  while (true) {
+    const open = text.indexOf(FROM_REQUEST_OPEN, cursor)
+    if (open === -1) return result + text.slice(cursor)
+    const close = text.indexOf(FROM_REQUEST_CLOSE, open + FROM_REQUEST_OPEN.length)
+    if (close === -1) {
+      throw new Error(`llm-replay: fromRequest placeholder is unterminated in ${JSON.stringify(text)}`)
+    }
+    const pattern = text.slice(open + FROM_REQUEST_OPEN.length, close)
+    result += text.slice(cursor, open) + resolveFromRequest(pattern, corpus)
+    cursor = close + FROM_REQUEST_CLOSE.length
+  }
+}
+
+/** Deep-copy one JSON-shaped value with scripted placeholders resolved. */
+function substituteValue(value: unknown, corpus: string): unknown {
+  if (typeof value === 'string') {
+    return value.includes(FROM_REQUEST_OPEN) ? substituteString(value, corpus) : value
+  }
+  if (Array.isArray(value)) return value.map(item => substituteValue(item, corpus))
+  if (value !== null && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, substituteValue(item, corpus)]))
+  }
+  return value
+}
+
+/**
+ * Resolve every `{{fromRequest:<regex>}}` placeholder in one scripted entry
+ * against the live request. The corpus is every string leaf of the request
+ * messages joined by newlines; the pattern's LAST corpus match wins and its
+ * first capture group (or, without one, the whole match) substitutes in place.
+ * Scenario sidecars use this to script arguments no static file can know,
+ * such as a randomly minted goal id the model must echo back. A pattern that
+ * matches nothing, an invalid pattern, and an unterminated placeholder each
+ * fail loud. Patterns cannot contain `}}` — the first `}}` ends the placeholder.
+ * @param entry - the scripted entry about to replay.
+ * @param messages - the live request messages searched by the placeholders.
+ * @returns the entry itself when no placeholder appears, else a resolved deep copy.
+ */
+export function resolveScriptedEntry(entry: ReplayEntry, messages: GenerateOptions['messages']): ReplayEntry {
+  if (!JSON.stringify(entry).includes(FROM_REQUEST_OPEN)) return entry
+  const leaves: string[] = []
+  collectStrings(messages, leaves)
+  return substituteValue(entry, leaves.join('\n')) as ReplayEntry
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null && !Array.isArray(value)
 }
@@ -583,7 +667,7 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): ReplayHand
           + `but its script has only ${boundState.entries.length}; re-record the scenario`,
         )
       }
-      yield* replayEntry(entry, options.signal, paceMs)
+      yield* replayEntry(resolveScriptedEntry(entry, options.messages), options.signal, paceMs)
     })()
   }
   const providers = config.providers ?? []