Merge remote-tracking branch 'origin/master' into codex/simp-session-dead-surface

# Conflicts:
#	docs/cordis-catalog/services.md
#	packages/core/session/README.md
#	packages/core/session/tests/derived-cache.spec.ts
#	packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts
#	packages/session-persistence/session-persistence/src/coordinator.ts
#	packages/session-persistence/session-persistence/src/index.ts

packages/session-persistence/session-persistence-jsonl/src/format.ts

@@ -72,19 +72,13 @@ function isHeaderLine(value: unknown): value is HeaderLine {
 }
 
 /**
- * Encode an arbitrary string as a single safe path segment, injectively over
- * ALL JS (UTF-16) strings — including lone surrogates. A {@link SessionId} is
- * an unvalidated branded string, so this neutralizes `../`, absolute paths,
- * NUL, and separators before any filesystem use.
+ * Encode an arbitrary string as a single safe path segment, injectively over ALL JS (UTF-16)
+ * strings — including lone surrogates. A {@link SessionId} is an unvalidated branded string,
+ * so this neutralizes `../`, absolute paths, NUL, and separators before any filesystem use.
+ * Safe code units remain literal; every other unit, including `~`, becomes
+ * `~XXXX`. Operating on code units preserves lone surrogates, while special-
+ * casing `.` and `..` prevents traversal by an otherwise safe whole segment.
  *
- * Each UTF-16 code unit is either kept literal (the safe set `[A-Za-z0-9_-]`)
- * or escaped as `~XXXX` (its 4-hex-digit code unit). `~` is itself escaped, so
- * the mapping is injective and reversible: distinct inputs never collide. We
- * iterate code UNITS (`charCodeAt`), not code points, so a lone surrogate
- * escapes to a distinct `~XXXX` instead of being normalized to U+FFFD (which
- * `Buffer.from(…, 'utf8')` would do, breaking injectivity). `.` is in the safe
- * set for readability but the whole-segment tokens `.`/`..` are escaped so they
- * can never traverse.
  * @param raw - the string to encode; must be non-empty (throws on `''`).
  * @returns the escaped single path segment, decodable back to `raw`.
  */
@@ -142,38 +136,18 @@ export function eventLine(event: SessionEvent): string {
 
 /**
  * Parse a JSONL log buffer into its preserved event prefix (the header is line
- * 0). Returns the longest prefix of complete, seq-contiguous events plus the
- * byte offset of the end of the last preserved line (`committedBytes`).
+ * 0). Fully written events in an interrupted final turn remain part of the
+ * prefix. The first unparsable record or seq gap after the last `turn/end`
+ * marks a tolerated torn tail; the same hole in the committed region rejects.
  *
- * A crash can leave a durable log whose final turn never closed: real,
- * fully-written events sit after the last `turn/end`. Those are PRESERVED (a
- * single turn can be huge in a long-horizon task — truncating it would destroy
- * real work); the backend closes the orphaned open turn with a synthetic
- * `turn/end {kind:'interrupted'}` on reload (the session-persistence RFC). Only a TORN trailing
- * fragment — a final line never fully flushed (no newline, unparseable, or a
- * seq gap) — is excluded; it bounds the preserved region. A parse error or seq
- * gap AT OR BEFORE the last committed `turn/end` is committed-data corruption
- * and makes the session unloadable (throws).
- *
- * This relies on the session-log invariant that every event lives inside a turn
- * (`Session.append` enforces it): only the final turn can be open, so the
- * preserved tail is at most one unclosed turn.
  * @param buffer - the raw bytes of the log file (header line first).
  * @returns the header, the preserved event prefix, and `committedBytes` — the
  *   byte offset the next append truncates any torn tail to.
  */
 export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionEvent[]; committedBytes: number } {
   const text = buffer.toString('utf8')
-  // Split into complete (newline-terminated) lines, tracking the byte offset of
-  // each line's end so the truncation point is exact (multi-byte chars make the
-  // char offset differ from the byte offset). A trailing line with no newline is
-  // an uncommitted crash fragment and is ignored — it is below the last
-  // turn/end by construction (the loop only flushes whole lines).
-  //
-  // Track the byte offset with a RUNNING accumulator (`endByte`), adding each
-  // line's byte length as we go. Recomputing `Buffer.byteLength(text.slice(0, i))`
-  // per newline would rescan the whole prefix every time — O(n²) over a long
-  // log (one assistant/chunk line per token makes that pathological).
+  // Track complete lines by byte offset: a non-newline tail is torn and ignored,
+  // and a running counter avoids rescanning a long multi-byte log.
   const lines: { text: string; endByte: number }[] = []
   let start = 0
   let byteOffset = 0
@@ -201,14 +175,8 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
   }
   const headerLine = parsedHeader
 
-  // Find the committed region: the prefix up to and including the LAST complete
-  // `turn/end` in the WHOLE log. Two passes so a crash tail after the last
-  // turn/end is tolerated, but corruption/gaps AT OR BEFORE the last committed
-  // turn/end make the log unloadable (committed data must never be silently
-  // dropped).
-  //
-  // Pass 1: parse every line that parses, recording (parsedOk, seq, isTurnEnd,
-  // endByte) per line index. Lines that fail to parse are holes.
+  // Parse every complete record first so the last valid `turn/end` determines
+  // whether an earlier hole is committed corruption or an uncommitted tail.
   interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
   const parsed: Parsed[] = eventEntries.map((entry) => {
     try {
@@ -226,18 +194,8 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
     if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
   }
 
-  // Walk the longest PREFIX of complete, seq-contiguous, parseable event lines
-  // (line i is a parsed event with seq === i). This is the preservable region:
-  // it includes any fully-written events of an interrupted final turn AFTER the
-  // last turn/end — those are real, durably-written work and must NOT be
-  // truncated (a single turn can be huge in a long-horizon task; the orphaned
-  // open turn is closed with a synthetic turn/end on reload, not discarded —
-  // the session-persistence RFC). The walk stops at the first hole (unparseable line or seq gap):
-  //   - if that hole is AT OR BEFORE the last committed turn/end, committed data
-  //     was damaged → the session is unloadable (throw);
-  //   - if it is AFTER (or there is no committed turn/end yet), it is the
-  //     tolerated crash boundary — a torn final line never fully flushed — and
-  //     it simply bounds the preserved tail.
+  // Preserve the contiguous prefix, including a complete interrupted turn;
+  // holes through the last committed boundary throw, while later holes stop.
   const preserved: SessionEvent[] = []
   for (let i = 0; i < parsed.length; i++) {
     const p = parsed[i]

packages/session-persistence/session-persistence-jsonl/src/index.ts

@@ -1,19 +1,7 @@
 /**
- * JSONL durable session-persistence backend (`@deepseek-ai/dsh-session-persistence-jsonl`).
- *
- * One append-only `.jsonl` event log per session (a header line then one
- * `SessionEvent` per line, verbatim including `assistant/chunk` so `seq` stays
- * contiguous), with lazy materialization (no file until the first `append`),
- * atomic first write, and load-time repair of a never-committed crash tail.
- *
- * The backend supplies ONLY the file-bytes storage primitives (the
- * {@link PersistenceBackend} hooks below); all the write-path orchestration
- * (the `session/event` → buffer → `session/flush` drain, per-session
- * serialization, write cursors, fork-seed persistence, HMR live-adoption,
- * crash-repair sequencing, dispose quiescence) lives in the backend-agnostic
- * {@link PersistenceCoordinator} this class composes. The four public
- * {@link SessionPersistence} methods delegate to the coordinator.
- *
+ * JSONL durable session-persistence backend. It stores a header and contiguous
+ * events in one append-only file per session, and delegates orchestration to
+ * {@link PersistenceCoordinator}.
  * @module @deepseek-ai/dsh-session-persistence-jsonl
  */
 
@@ -41,13 +29,7 @@ export interface Config {
   root: string
 }
 
-/**
- * Whether `error` is a "no such file/directory" (`ENOENT`) failure — the ONLY
- * filesystem error that legitimately means "this session/root is absent" for a
- * durable backend. Any OTHER error (`EACCES`, `ENOTDIR`, transient I/O) must
- * surface rather than be silently reported as absence. (A NodeJS filesystem
- * rejection carries a string `code`.)
- */
+/** Whether a filesystem error means absence; every non-ENOENT failure must surface. */
 function isENOENT(error: unknown): boolean {
   return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT'
 }
@@ -65,13 +47,9 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
   })
 
   /**
-   * Backend label for the coordinator's dispose-failure AggregateError and
-   * effect name. NOTE: this intentionally shadows cordis `Service.name` (which
-   * the base sets to `'sessionPersistence'`). The service is registered under the
-   * fixed key the Service constructor captured (`reflect.provide('sessionPersistence', …)`),
-   * not via `this.name`, so overwriting the instance field with the backend label
-   * does not affect `ctx.sessionPersistence` resolution — it only relabels the
-   * dispose diagnostics, which is exactly what {@link PersistenceBackend.name} is for.
+   * Backend label for coordinator diagnostics and effects. It shadows
+   * `Service.name` without changing the service key captured by the base
+   * constructor.
    */
   override readonly name = 'session-persistence-jsonl'
 
@@ -80,10 +58,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
 
   constructor(ctx: Context, public config: Config) {
     super(ctx)
-    // Resolve the configured root to an ABSOLUTE path ONCE, here. A relative root
-    // would otherwise re-resolve against `process.cwd()` at every later
-    // readdir/open — so if any plugin or test changed cwd between create, append,
-    // and load, one session's files could split across directories.
+    // Resolve once so later process.cwd() changes cannot split one backend across roots.
     this.root = resolve(config.root)
     this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
   }
@@ -105,16 +80,13 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
     return this.coordinator.load(id)
   }
 
-  // `list` is BOTH the public service method and the PersistenceBackend hook —
-  // one method, the bucket walk below. The coordinator adds no orchestration for
-  // listing (no per-id serialization, no cursor), so it would just call back into
-  // this same method; routing it through the coordinator would recurse. Defined
-  // once, in the "PersistenceBackend hooks" section.
+  // One method serves both public `list` and the backend hook; delegating it to
+  // the coordinator would call this hook recursively.
 
   /* jscpd:ignore-end */
   // --- PersistenceBackend hooks (the file-bytes storage primitives) ---
 
-  /** Read a stored prefix by id across ALL cwd buckets (cwd unknown). */
+  /** Read a stored prefix by id across all cwd buckets when cwd is unknown. */
   async loadStored(id: SessionId): Promise<StoredPrefix<number> | undefined> {
     const file = await this.findLog(id)
     if (file === undefined) return undefined
@@ -122,11 +94,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
   }
 
   /**
-   * Read a stored prefix SCOPED to `cwd` (HMR live-adoption must not cross cwd).
-   * `undefined` is the DEFINITE "no-cwd" bucket, NOT "unknown" — a live session
-   * with no cwd may only adopt a persisted no-cwd log, never a same-id log that
-   * lives in some other cwd bucket. So this looks at exactly `logPath(cwd)`
-   * (which maps `undefined` → the `_no-cwd` bucket), never the all-buckets scan.
+   * Read a stored prefix within one cwd for HMR adoption. `undefined` names the
+   * no-cwd bucket rather than an unknown cwd, so this never scans other buckets.
    */
   async loadLive(id: SessionId, cwd: string | undefined): Promise<StoredPrefix<number> | undefined> {
     const path = logPath(this.root, cwd, id)
@@ -135,10 +104,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
   }
 
   /**
-   * Read and scan a session's log file into a {@link StoredPrefix}. Folds the
-   * torn-tail comparison HERE so the `tornMarker` is the byte offset to truncate
-   * to (or `undefined` when nothing is torn) — the coordinator never sees the
-   * raw byteLength.
+   * Read a stored prefix and convert torn-tail state to the byte offset the
+   * coordinator can round-trip without knowing the file format.
    */
   private async readPrefix(path: string): Promise<StoredPrefix<number>> {
     const buffer = await readFile(path)
@@ -174,9 +141,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
     const metas: SessionHeader[] = []
     for (const dir of await this.listCwdDirs()) {
       for (const name of await this.listJsonl(dir)) {
-        // Read ONLY the header line, not the whole log: a session picker must
-        // scale with the number of sessions, not the total size of every
-        // conversation (the log persists every assistant/chunk verbatim).
+        // Read only headers so listing scales with session count, not log size.
         const first = await this.readFirstLine(`${dir}/${name}`)
         if (first === undefined) continue // empty/half-written file
         const meta = parseHeaderMeta(first)
@@ -197,10 +162,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
     await mkdir(dir, { recursive: true, mode: 0o700 })
     await this.syncDir(this.root)
     const finalPath = logPath(this.root, meta.cwd, meta.id)
-    // Never rename over an existing committed log: materialize is the FIRST write
-    // of a session the backend believes is new. A file here means a different
-    // session shares this id on disk — reject loudly. (createCore already guards
-    // the create path, so this is unreachable-in-practice TOCTOU defense.)
+    // Materialization is the first write; an existing log is an id collision.
     /* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
     if (await this.exists(finalPath)) {
       throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
@@ -217,28 +179,22 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
     } finally {
       await handle.close()
     }
-    // Publish via link()+unlink(), NOT rename(): link fails with EEXIST if the
-    // final path already exists, so two processes materializing the same id
-    // concurrently cannot clobber each other. rename() would silently overwrite.
+    // Publish with link()+unlink(): unlike rename(), link fails if another
+    // process materialized the same id first.
     let linked = false
     try {
       await link(tmp, finalPath)
       linked = true
     } finally {
-      // If link FAILED, the temp is the only reference and must be removed before
-      // the original error propagates. If it SUCCEEDED, defer temp cleanup to
-      // AFTER the publish is durable (below) so a temp-rm failure can never reject
-      // a session whose log already published.
+      // Remove an unpublished temp on failure. After publication, defer cleanup
+      // until the directory entry is durable so cleanup cannot reject a live log.
       /* v8 ignore next -- link failure is the TOCTOU/IO race guarded above; not reachable in test */
       if (!linked) await rm(tmp, { force: true })
     }
-    // link() succeeded — the log is published. fsync the directory so the new
-    // entry survives a power loss: the new link is not crash-durable until the
-    // parent directory's metadata is synced.
+    // The published link becomes crash-durable only after its directory fsync.
     await this.syncDir(dir)
-    // Best-effort temp cleanup: the log is already published and durable, so a
-    // failure to remove the (now-redundant) temp hard link must NOT reject the
-    // append. Swallow only the rm failure; nothing else of consequence runs here.
+    // Best-effort temp cleanup: the log is already published and durable, so a failure to
+    // remove the (now-redundant) temp hard link must not reject the append.
     try {
       await rm(tmp, { force: true })
     } catch {
@@ -257,11 +213,9 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
   }
 
   /**
-   * Append event lines at EOF and fsync. On a write/sync failure AFTER the kernel
-   * accepted some bytes (ENOSPC, an fsync error), truncate the file back to its
-   * pre-append size before rethrowing: the cursor is unchanged, so the batch will
-   * be retried, and without this rollback the retry would append AFTER the partial
-   * bytes — producing duplicate seqs that make `scanLog` see a gap.
+   * Append and fsync event lines. On a partial write or sync failure, restore the
+   * previous size before rethrowing because the unchanged cursor will retry the
+   * batch; leaving partial bytes would create duplicate sequence numbers.
    */
   private async appendLines(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> {
     const path = logPath(this.root, meta.cwd, meta.id)
@@ -323,10 +277,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
   }
 
   /**
-   * Find a session's log file by id across ALL cwd buckets — the any-cwd scan
-   * for `loadStored` (resume identifies a session by id alone). The cwd-scoped
-   * lookup (`loadLive`) does NOT use this; it goes straight to `logPath(cwd)` so
-   * a no-cwd session can't match a real-cwd bucket.
+   * Find a session by id across cwd buckets for resume. Cwd-scoped HMR adoption
+   * bypasses this scan so a no-cwd session cannot claim another bucket.
    */
   private async findLog(id: SessionId): Promise<{ path: string; cwd: string | undefined } | undefined> {
     const target = encodeSegment(id) + '.jsonl'
@@ -347,9 +299,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
       const entries = await readdir(this.root, { withFileTypes: true })
       return entries.filter(e => e.isDirectory()).map(e => `${this.root}/${e.name}`)
     } catch (error) {
-      // ENOENT = the root has not been created yet → genuinely no sessions. Any
-      // other error (EACCES, ENOTDIR, transient I/O) must NOT be reported as "no
-      // sessions" — a durable backend cannot silently pretend state is absent.
+      // Only an absent root means no sessions; rethrow every other I/O failure.
       if (isENOENT(error)) return []
       throw error
     }