Reorganize packages into a modular hierarchy
Move the 18 flat packages/<name> packages into role-grouped dirs: core/, llm/, bash/, session-persistence/, ui/, support/. Group dirs are pure containers; each package keeps its @deepseek-ai/dsh-* name. Collapse the per-package tsconfig paths maps (base + typecheck) into one @deepseek-ai/dsh-* wildcard with a candidate per group, and derive the publint list from the hierarchy. Update all depth-coupled globs/configs (workspace, tsdown, vitest, eslint, knip, tsconfig includes/refs, per-package tsconfigs, generators, doc-script scopes, type-equiv manifest) and the cross-package/script relative imports in tests. Fix doc-typecheck's workspacePaths() to parse tsconfig JSONC via the TypeScript API instead of a regex comment-strip, which corrupted the new wildcard `/*/` path candidates. WIP: doc cross-links and package/RFC docs still to update.
This commit is contained in:
Tianyi Cui
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* On-disk format helpers for the JSONL session-persistence backend: path
|
||||
* sanitization (a {@link SessionId} is an unvalidated branded string, so it
|
||||
* MUST be encoded before use in a path — no traversal, no collision), the
|
||||
* per-cwd directory layout, header-line (de)serialization, and the
|
||||
* truncation-repair offset computation.
|
||||
*
|
||||
* @module dsh-session-persistence-jsonl/format
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import { join } from 'node:path'
|
||||
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
|
||||
|
||||
/**
|
||||
* The first line of a session's `.jsonl` file: the immutable
|
||||
* {@link SessionHeader} tagged as a `session` record so a reader can tell it
|
||||
* apart from an event line.
|
||||
*/
|
||||
export interface HeaderLine {
|
||||
type: 'session'
|
||||
version: number
|
||||
id: SessionId
|
||||
createdAt: number
|
||||
cwd?: string
|
||||
parentSession?: SessionId
|
||||
}
|
||||
|
||||
/** Build the header line object from a {@link SessionHeader}. */
|
||||
export function toHeaderLine(header: SessionHeader): HeaderLine {
|
||||
return {
|
||||
type: 'session',
|
||||
version: header.version,
|
||||
id: header.id,
|
||||
createdAt: header.createdAt,
|
||||
...header.cwd !== undefined ? { cwd: header.cwd } : {},
|
||||
...header.parentSession !== undefined ? { parentSession: header.parentSession } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse a header line back into a {@link SessionHeader}. */
|
||||
export function fromHeaderLine(line: HeaderLine): SessionHeader {
|
||||
return {
|
||||
version: line.version,
|
||||
id: line.id,
|
||||
createdAt: line.createdAt,
|
||||
...line.cwd !== undefined ? { cwd: line.cwd } : {},
|
||||
...line.parentSession !== undefined ? { parentSession: line.parentSession } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Type guard: a parsed first line is a well-formed session header. */
|
||||
function isHeaderLine(value: unknown): value is HeaderLine {
|
||||
return (
|
||||
typeof value === 'object' && value !== null
|
||||
&& (value as { type?: unknown }).type === 'session'
|
||||
&& typeof (value as { version?: unknown }).version === 'number'
|
||||
&& typeof (value as { id?: unknown }).id === 'string'
|
||||
&& typeof (value as { createdAt?: unknown }).createdAt === 'number'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an arbitrary string as a single safe path segment, injectively over
|
||||
* ALL JS (UTF-16) strings — including lone surrogates. A {@link SessionId} is
|
||||
* an unvalidated branded string, so this neutralizes `../`, absolute paths,
|
||||
* NUL, and separators before any filesystem use.
|
||||
*
|
||||
* Each UTF-16 code unit is either kept literal (the safe set `[A-Za-z0-9_-]`)
|
||||
* or escaped as `~XXXX` (its 4-hex-digit code unit). `~` is itself escaped, so
|
||||
* the mapping is injective and reversible: distinct inputs never collide. We
|
||||
* iterate code UNITS (`charCodeAt`), not code points, so a lone surrogate
|
||||
* escapes to a distinct `~XXXX` instead of being normalized to U+FFFD (which
|
||||
* `Buffer.from(…, 'utf8')` would do, breaking injectivity). `.` is in the safe
|
||||
* set for readability but the whole-segment tokens `.`/`..` are escaped so they
|
||||
* can never traverse.
|
||||
*/
|
||||
export function encodeSegment(raw: string): string {
|
||||
if (raw.length === 0) throw new Error('cannot encode an empty path segment')
|
||||
if (raw === '.') return '~002E'
|
||||
if (raw === '..') return '~002E~002E'
|
||||
let out = ''
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const code = raw.charCodeAt(i)
|
||||
const ch = String.fromCharCode(code)
|
||||
if (ch !== '~' && /^[A-Za-z0-9._-]$/.test(ch)) {
|
||||
out += ch
|
||||
} else {
|
||||
out += '~' + code.toString(16).toUpperCase().padStart(4, '0')
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* The directory a session's files live in: the configured root, then a per-cwd
|
||||
* subdirectory so sessions group by project. The cwd subdir is a stable hash
|
||||
* (short, collision-resistant, filesystem-safe) plus an encoded suffix for
|
||||
* readability; sessions without a cwd go in a shared `_no-cwd` bucket.
|
||||
*/
|
||||
export function sessionDir(root: string, cwd: string | undefined): string {
|
||||
if (cwd === undefined) return join(root, '_no-cwd')
|
||||
const hash = createHash('sha256').update(cwd).digest('hex').slice(0, 12)
|
||||
return join(root, `cwd-${hash}`)
|
||||
}
|
||||
|
||||
/** The append-only event-log file path for a session. */
|
||||
export function logPath(root: string, cwd: string | undefined, id: SessionId): string {
|
||||
return join(sessionDir(root, cwd), `${encodeSegment(id)}.jsonl`)
|
||||
}
|
||||
|
||||
/** Serialize one event as a JSONL line (no trailing newline). */
|
||||
export function eventLine(event: SessionEvent): string {
|
||||
return JSON.stringify(event)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a JSONL log buffer into its preserved event prefix (the header is line
|
||||
* 0). Returns the longest prefix of complete, seq-contiguous events plus the
|
||||
* byte offset of the end of the last preserved line (`committedBytes`).
|
||||
*
|
||||
* A crash can leave a durable log whose final turn never closed: real,
|
||||
* fully-written events sit after the last `turn/end`. Those are PRESERVED (a
|
||||
* single turn can be huge in a long-horizon task — truncating it would destroy
|
||||
* real work); the backend closes the orphaned open turn with a synthetic
|
||||
* `turn/end {kind:'interrupted'}` on reload (the session-persistence RFC). Only a TORN trailing
|
||||
* fragment — a final line never fully flushed (no newline, unparseable, or a
|
||||
* seq gap) — is excluded; it bounds the preserved region. A parse error or seq
|
||||
* gap AT OR BEFORE the last committed `turn/end` is committed-data corruption
|
||||
* and makes the session unloadable (throws).
|
||||
*
|
||||
* This relies on the session-log invariant that every event lives inside a turn
|
||||
* (`Session.append` enforces it): only the final turn can be open, so the
|
||||
* preserved tail is at most one unclosed turn.
|
||||
*/
|
||||
export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionEvent[]; committedBytes: number } {
|
||||
const text = buffer.toString('utf8')
|
||||
// Split into complete (newline-terminated) lines, tracking the byte offset of
|
||||
// each line's end so the truncation point is exact (multi-byte chars make the
|
||||
// char offset differ from the byte offset). A trailing line with no newline is
|
||||
// an uncommitted crash fragment and is ignored — it is below the last
|
||||
// turn/end by construction (the loop only flushes whole lines).
|
||||
//
|
||||
// Track the byte offset with a RUNNING accumulator (`endByte`), adding each
|
||||
// line's byte length as we go. Recomputing `Buffer.byteLength(text.slice(0, i))`
|
||||
// per newline would rescan the whole prefix every time — O(n²) over a long
|
||||
// log (one assistant/chunk line per token makes that pathological).
|
||||
const lines: { text: string; endByte: number }[] = []
|
||||
let start = 0
|
||||
let byteOffset = 0
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (text[i] === '\n') {
|
||||
const lineText = text.slice(start, i)
|
||||
byteOffset += Buffer.byteLength(lineText, 'utf8') + 1 // +1 for the '\n' (a 1-byte char)
|
||||
lines.push({ text: lineText, endByte: byteOffset })
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
|
||||
const [headerEntry, ...eventEntries] = lines
|
||||
if (headerEntry === undefined) throw new Error('empty or header-less session log')
|
||||
|
||||
// Line 0 is the header.
|
||||
let parsedHeader: unknown
|
||||
try {
|
||||
parsedHeader = JSON.parse(headerEntry.text)
|
||||
} catch {
|
||||
throw new Error('corrupt session log: header line is not valid JSON')
|
||||
}
|
||||
if (!isHeaderLine(parsedHeader)) {
|
||||
throw new Error('corrupt session log: first line is not a session header')
|
||||
}
|
||||
const headerLine = parsedHeader
|
||||
|
||||
// Find the committed region: the prefix up to and including the LAST complete
|
||||
// `turn/end` in the WHOLE log. Two passes so a crash tail after the last
|
||||
// turn/end is tolerated, but corruption/gaps AT OR BEFORE the last committed
|
||||
// turn/end make the log unloadable (committed data must never be silently
|
||||
// dropped).
|
||||
//
|
||||
// Pass 1: parse every line that parses, recording (parsedOk, seq, isTurnEnd,
|
||||
// endByte) per line index. Lines that fail to parse are holes.
|
||||
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
|
||||
const parsed: Parsed[] = eventEntries.map((entry) => {
|
||||
try {
|
||||
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
|
||||
} catch {
|
||||
return { ok: false, endByte: entry.endByte }
|
||||
}
|
||||
})
|
||||
|
||||
// The last index (into eventEntries) that is a valid `turn/end` — the last
|
||||
// fully-committed boundary (the loop flushes only at turn/end).
|
||||
let lastTurnEnd = -1
|
||||
for (let i = parsed.length - 1; i >= 0; i--) {
|
||||
const p = parsed[i]
|
||||
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
|
||||
}
|
||||
|
||||
// Walk the longest PREFIX of complete, seq-contiguous, parseable event lines
|
||||
// (line i is a parsed event with seq === i). This is the preservable region:
|
||||
// it includes any fully-written events of an interrupted final turn AFTER the
|
||||
// last turn/end — those are real, durably-written work and must NOT be
|
||||
// truncated (a single turn can be huge in a long-horizon task; the orphaned
|
||||
// open turn is closed with a synthetic turn/end on reload, not discarded —
|
||||
// the session-persistence RFC). The walk stops at the first hole (unparseable line or seq gap):
|
||||
// - if that hole is AT OR BEFORE the last committed turn/end, committed data
|
||||
// was damaged → the session is unloadable (throw);
|
||||
// - if it is AFTER (or there is no committed turn/end yet), it is the
|
||||
// tolerated crash boundary — a torn final line never fully flushed — and
|
||||
// it simply bounds the preserved tail.
|
||||
const preserved: SessionEvent[] = []
|
||||
for (let i = 0; i < parsed.length; i++) {
|
||||
const p = parsed[i]
|
||||
if (!p?.ok || p.event === undefined) {
|
||||
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
|
||||
break // torn tail fragment after the last turn/end — stop, tolerate
|
||||
}
|
||||
if (p.event.seq !== i) {
|
||||
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
|
||||
break // gap after the last turn/end — torn tail, stop
|
||||
}
|
||||
preserved.push(p.event)
|
||||
}
|
||||
|
||||
// committedBytes = end of the last PRESERVED line (header if none): the next
|
||||
// append truncates any torn bytes past this point before writing the
|
||||
// synthetic closers + new events.
|
||||
const lastPreserved = parsed[preserved.length - 1]
|
||||
const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte
|
||||
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse just the header line of a log into a {@link SessionHeader}, or
|
||||
* `undefined` if it is missing/not a header. Used by `list()` to read session
|
||||
* metadata WITHOUT parsing the whole log: a session picker scales with the
|
||||
* number of sessions, not the total size of every conversation.
|
||||
*/
|
||||
export function parseHeaderMeta(firstLine: string): SessionHeader | undefined {
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(firstLine)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
if (!isHeaderLine(parsed)) return undefined
|
||||
return fromHeaderLine(parsed)
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
/**
|
||||
* JSONL durable session-persistence backend (`@deepseek-ai/dsh-session-persistence-jsonl`).
|
||||
*
|
||||
* One append-only `.jsonl` event log per session (a header line then one
|
||||
* `SessionEvent` per line, verbatim including `assistant/chunk` so `seq` stays
|
||||
* contiguous), with lazy materialization (no file until the first `append`),
|
||||
* atomic first write, and load-time repair of a never-committed crash tail.
|
||||
*
|
||||
* The backend supplies ONLY the file-bytes storage primitives (the
|
||||
* {@link PersistenceBackend} hooks below); all the write-path orchestration
|
||||
* (the `session/event` → buffer → `session/flush` drain, per-session
|
||||
* serialization, write cursors, fork-seed persistence, HMR live-adoption,
|
||||
* crash-repair sequencing, dispose quiescence) lives in the backend-agnostic
|
||||
* {@link PersistenceCoordinator} this class composes. The six public
|
||||
* {@link SessionPersistence} methods delegate to the coordinator.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-persistence-jsonl
|
||||
*/
|
||||
|
||||
import { Context } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import { open, mkdir, readFile, readdir, link, rm, truncate } from 'node:fs/promises'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import {
|
||||
SessionPersistence, PersistenceCoordinator,
|
||||
type PersistenceBackend, type StoredPrefix,
|
||||
} from '@deepseek-ai/dsh-session-persistence'
|
||||
import type { Session, SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
|
||||
} from './format.ts'
|
||||
|
||||
export interface Config {
|
||||
/**
|
||||
* Root directory for all session files. Required (no default): a default of
|
||||
* `process.cwd()` would scatter session files as the process's cwd changes
|
||||
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
|
||||
*/
|
||||
root: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `error` is a "no such file/directory" (`ENOENT`) failure — the ONLY
|
||||
* filesystem error that legitimately means "this session/root is absent" for a
|
||||
* durable backend. Any OTHER error (`EACCES`, `ENOTDIR`, transient I/O) must
|
||||
* surface rather than be silently reported as absence. (A NodeJS filesystem
|
||||
* rejection carries a string `code`.)
|
||||
*/
|
||||
function isENOENT(error: unknown): boolean {
|
||||
return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT'
|
||||
}
|
||||
|
||||
/**
|
||||
* The JSONL persistence backend. Load as a plugin; it registers as
|
||||
* `ctx.sessionPersistence` and (via the coordinator) installs the write-path
|
||||
* listeners. Its torn-tail marker is the byte offset to truncate the log to.
|
||||
*/
|
||||
export class SessionPersistenceJsonl extends SessionPersistence implements PersistenceBackend<number> {
|
||||
static inject = ['sessions']
|
||||
|
||||
static Config: z<Config> = z.object({
|
||||
root: z.string().required(),
|
||||
})
|
||||
|
||||
/**
|
||||
* Backend label for the coordinator's dispose-failure AggregateError and
|
||||
* effect name. NOTE: this intentionally shadows cordis `Service.name` (which
|
||||
* the base sets to `'sessionPersistence'`). The service is registered under the
|
||||
* fixed key the Service constructor captured (`reflect.provide('sessionPersistence', …)`),
|
||||
* not via `this.name`, so overwriting the instance field with the backend label
|
||||
* does not affect `ctx.sessionPersistence` resolution — it only relabels the
|
||||
* dispose diagnostics, which is exactly what {@link PersistenceBackend.name} is for.
|
||||
*/
|
||||
override readonly name = 'session-persistence-jsonl'
|
||||
|
||||
private root: string
|
||||
private coordinator: PersistenceCoordinator<number>
|
||||
|
||||
constructor(ctx: Context, public config: Config) {
|
||||
super(ctx)
|
||||
// Resolve the configured root to an ABSOLUTE path ONCE, here. A relative root
|
||||
// would otherwise re-resolve against `process.cwd()` at every later
|
||||
// readdir/open — so if any plugin or test changed cwd between create, append,
|
||||
// and load, one session's files could split across directories.
|
||||
this.root = resolve(config.root)
|
||||
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
|
||||
}
|
||||
|
||||
// --- SessionPersistence service surface (delegated to the coordinator) ---
|
||||
|
||||
create(meta: SessionHeader): Promise<void> {
|
||||
return this.coordinator.create(meta)
|
||||
}
|
||||
|
||||
append(id: SessionId, events: readonly SessionEvent[]): Promise<void> {
|
||||
return this.coordinator.append(id, events)
|
||||
}
|
||||
|
||||
load(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> {
|
||||
return this.coordinator.load(id)
|
||||
}
|
||||
|
||||
has(id: SessionId): Promise<boolean> {
|
||||
return this.coordinator.has(id)
|
||||
}
|
||||
|
||||
delete(id: SessionId): Promise<void> {
|
||||
return this.coordinator.delete(id)
|
||||
}
|
||||
|
||||
// `list` is BOTH the public service method and the PersistenceBackend hook —
|
||||
// one method, the bucket walk below. The coordinator adds no orchestration for
|
||||
// listing (no per-id serialization, no cursor), so it would just call back into
|
||||
// this same method; routing it through the coordinator would recurse. Defined
|
||||
// once, in the "PersistenceBackend hooks" section.
|
||||
|
||||
/**
|
||||
* The per-session init promises, exposed for white-box tests that await a
|
||||
* specific session's onCreated (there is no public API to await one init).
|
||||
*/
|
||||
get inits(): Map<Session, Promise<void>> {
|
||||
return this.coordinator.inits
|
||||
}
|
||||
|
||||
// --- PersistenceBackend hooks (the file-bytes storage primitives) ---
|
||||
|
||||
/** Read a stored prefix by id across ALL cwd buckets (cwd unknown). */
|
||||
async loadStored(id: SessionId): Promise<StoredPrefix<number> | undefined> {
|
||||
const file = await this.findLog(id)
|
||||
if (file === undefined) return undefined
|
||||
return this.readPrefix(file.path)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a stored prefix SCOPED to `cwd` (HMR live-adoption must not cross cwd).
|
||||
* `undefined` is the DEFINITE "no-cwd" bucket, NOT "unknown" — a live session
|
||||
* with no cwd may only adopt a persisted no-cwd log, never a same-id log that
|
||||
* lives in some other cwd bucket. So this looks at exactly `logPath(cwd)`
|
||||
* (which maps `undefined` → the `_no-cwd` bucket), never the all-buckets scan.
|
||||
*/
|
||||
async loadLive(id: SessionId, cwd: string | undefined): Promise<StoredPrefix<number> | undefined> {
|
||||
const path = logPath(this.root, cwd, id)
|
||||
if (!await this.exists(path)) return undefined
|
||||
return this.readPrefix(path)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read and scan a session's log file into a {@link StoredPrefix}. Folds the
|
||||
* torn-tail comparison HERE so the `tornMarker` is the byte offset to truncate
|
||||
* to (or `undefined` when nothing is torn) — the coordinator never sees the
|
||||
* raw byteLength.
|
||||
*/
|
||||
private async readPrefix(path: string): Promise<StoredPrefix<number>> {
|
||||
const buffer = await readFile(path)
|
||||
const { meta, events, committedBytes } = scanLog(buffer)
|
||||
return {
|
||||
meta,
|
||||
events,
|
||||
...committedBytes < buffer.byteLength ? { tornMarker: committedBytes } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Durably append a batch, lazily materializing the file when not yet present. */
|
||||
async appendBatch(meta: SessionHeader, events: readonly SessionEvent[], isMaterialized: boolean): Promise<void> {
|
||||
if (isMaterialized) {
|
||||
await this.appendLines(meta, events)
|
||||
} else {
|
||||
await this.materialize(meta, events)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a crash repair durable: truncate the torn tail to `tornMarker` bytes (if
|
||||
* any), then append the synthetic `closers` (if any). Two fsync'd steps — the
|
||||
* seam does not require this to be atomic.
|
||||
*/
|
||||
async commitRepair(meta: SessionHeader, tornMarker: number | undefined, closers: readonly SessionEvent[]): Promise<void> {
|
||||
if (tornMarker !== undefined) await this.repair(meta, tornMarker)
|
||||
if (closers.length > 0) await this.appendLines(meta, closers)
|
||||
}
|
||||
|
||||
/** Remove a session's log file (the coordinator clears its in-memory state). */
|
||||
async deleteStored(id: SessionId): Promise<void> {
|
||||
const file = await this.findLog(id)
|
||||
if (file) await rm(file.path, { force: true })
|
||||
}
|
||||
|
||||
/** List all stored sessions' metadata (header line only — no full-log parse). */
|
||||
async list(): Promise<SessionHeader[]> {
|
||||
const metas: SessionHeader[] = []
|
||||
for (const dir of await this.listCwdDirs()) {
|
||||
for (const name of await this.listJsonl(dir)) {
|
||||
// Read ONLY the header line, not the whole log: a session picker must
|
||||
// scale with the number of sessions, not the total size of every
|
||||
// conversation (the log persists every assistant/chunk verbatim).
|
||||
const first = await this.readFirstLine(`${dir}/${name}`)
|
||||
if (first === undefined) continue // empty/half-written file
|
||||
const meta = parseHeaderMeta(first)
|
||||
if (meta === undefined) continue // not a session header
|
||||
metas.push(meta)
|
||||
}
|
||||
}
|
||||
return metas
|
||||
}
|
||||
|
||||
// --- materialization / append / repair (file mechanics) ---
|
||||
|
||||
/** Atomically write the header line + first batch (temp-write, fsync, rename). */
|
||||
private async materialize(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> {
|
||||
const dir = sessionDir(this.root, meta.cwd)
|
||||
await mkdir(this.root, { recursive: true, mode: 0o700 })
|
||||
await this.syncDir(dirname(this.root))
|
||||
await mkdir(dir, { recursive: true, mode: 0o700 })
|
||||
await this.syncDir(this.root)
|
||||
const finalPath = logPath(this.root, meta.cwd, meta.id)
|
||||
// Never rename over an existing committed log: materialize is the FIRST write
|
||||
// of a session the backend believes is new. A file here means a different
|
||||
// session shares this id on disk — reject loudly. (createCore already guards
|
||||
// the create path, so this is unreachable-in-practice TOCTOU defense.)
|
||||
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
|
||||
if (await this.exists(finalPath)) {
|
||||
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
|
||||
}
|
||||
const header = JSON.stringify(toHeaderLine(meta))
|
||||
const body = events.map(eventLine).join('\n')
|
||||
const content = header + '\n' + body + '\n'
|
||||
|
||||
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
|
||||
const handle = await open(tmp, 'wx', 0o600)
|
||||
try {
|
||||
await handle.writeFile(content)
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
// Publish via link()+unlink(), NOT rename(): link fails with EEXIST if the
|
||||
// final path already exists, so two processes materializing the same id
|
||||
// concurrently cannot clobber each other. rename() would silently overwrite.
|
||||
let linked = false
|
||||
try {
|
||||
await link(tmp, finalPath)
|
||||
linked = true
|
||||
} finally {
|
||||
// If link FAILED, the temp is the only reference and must be removed before
|
||||
// the original error propagates. If it SUCCEEDED, defer temp cleanup to
|
||||
// AFTER the publish is durable (below) so a temp-rm failure can never reject
|
||||
// a session whose log already published.
|
||||
/* v8 ignore next -- link failure is the TOCTOU/IO race guarded above; not reachable in test */
|
||||
if (!linked) await rm(tmp, { force: true })
|
||||
}
|
||||
// link() succeeded — the log is published. fsync the directory so the new
|
||||
// entry survives a power loss: the new link is not crash-durable until the
|
||||
// parent directory's metadata is synced.
|
||||
await this.syncDir(dir)
|
||||
// Best-effort temp cleanup: the log is already published and durable, so a
|
||||
// failure to remove the (now-redundant) temp hard link must NOT reject the
|
||||
// append. Swallow only the rm failure; nothing else of consequence runs here.
|
||||
try {
|
||||
await rm(tmp, { force: true })
|
||||
} catch {
|
||||
/* v8 ignore next -- redundant temp link; publish already durable, rm failure is an unreachable IO edge */
|
||||
}
|
||||
}
|
||||
|
||||
/** fsync a directory so a just-created/renamed entry inside it is crash-durable. */
|
||||
private async syncDir(dir: string): Promise<void> {
|
||||
const handle = await open(dir, 'r')
|
||||
try {
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append event lines at EOF and fsync. On a write/sync failure AFTER the kernel
|
||||
* accepted some bytes (ENOSPC, an fsync error), truncate the file back to its
|
||||
* pre-append size before rethrowing: the cursor is unchanged, so the batch will
|
||||
* be retried, and without this rollback the retry would append AFTER the partial
|
||||
* bytes — producing duplicate seqs that make `scanLog` see a gap.
|
||||
*/
|
||||
private async appendLines(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> {
|
||||
const path = logPath(this.root, meta.cwd, meta.id)
|
||||
const handle = await open(path, 'a')
|
||||
try {
|
||||
const { size: before } = await handle.stat()
|
||||
try {
|
||||
await handle.writeFile(events.map(eventLine).join('\n') + '\n')
|
||||
await handle.sync()
|
||||
} catch (error) {
|
||||
// Roll back whatever bytes landed so a retry starts from a clean EOF.
|
||||
await handle.truncate(before)
|
||||
await handle.sync()
|
||||
throw error
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/** Truncate the log file to `offset` bytes and fsync (discard the crash tail). */
|
||||
private async repair(meta: SessionHeader, offset: number): Promise<void> {
|
||||
const path = logPath(this.root, meta.cwd, meta.id)
|
||||
await truncate(path, offset)
|
||||
const handle = await open(path, 'r+')
|
||||
try {
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
// --- discovery helpers ---
|
||||
|
||||
/**
|
||||
* Read the first newline-terminated line of a file without loading the whole
|
||||
* file. Returns undefined if the file is empty or has no complete first line.
|
||||
* Reads in bounded chunks so a huge log costs only the header read.
|
||||
*/
|
||||
private async readFirstLine(path: string): Promise<string | undefined> {
|
||||
const handle = await open(path, 'r')
|
||||
try {
|
||||
const chunks: Buffer[] = []
|
||||
const buf = Buffer.alloc(8192)
|
||||
for (;;) {
|
||||
const { bytesRead } = await handle.read(buf, 0, buf.length, null)
|
||||
if (bytesRead === 0) return undefined // EOF with no newline → no complete line
|
||||
const slice = buf.subarray(0, bytesRead)
|
||||
const nl = slice.indexOf(0x0a)
|
||||
if (nl !== -1) {
|
||||
chunks.push(slice.subarray(0, nl))
|
||||
return Buffer.concat(chunks).toString('utf8')
|
||||
}
|
||||
chunks.push(Buffer.from(slice))
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a session's log file by id across ALL cwd buckets — the any-cwd scan
|
||||
* for `loadStored`/`deleteStored` (resume and removal identify a session by id
|
||||
* alone). The cwd-scoped lookup (`loadLive`) does NOT use this; it goes
|
||||
* straight to `logPath(cwd)` so a no-cwd session can't match a real-cwd bucket.
|
||||
*/
|
||||
private async findLog(id: SessionId): Promise<{ path: string; cwd: string | undefined } | undefined> {
|
||||
const target = encodeSegment(id) + '.jsonl'
|
||||
for (const dir of await this.listCwdDirs()) {
|
||||
const path = `${dir}/${target}`
|
||||
if (await this.exists(path)) {
|
||||
// Recover the cwd from the header so the caller has the session's bucket.
|
||||
const { meta } = scanLog(await readFile(path))
|
||||
return { path, cwd: meta.cwd }
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** The cwd-bucket directories under the root (absolute paths). */
|
||||
private async listCwdDirs(): Promise<string[]> {
|
||||
try {
|
||||
const entries = await readdir(this.root, { withFileTypes: true })
|
||||
return entries.filter(e => e.isDirectory()).map(e => `${this.root}/${e.name}`)
|
||||
} catch (error) {
|
||||
// ENOENT = the root has not been created yet → genuinely no sessions. Any
|
||||
// other error (EACCES, ENOTDIR, transient I/O) must NOT be reported as "no
|
||||
// sessions" — a durable backend cannot silently pretend state is absent.
|
||||
if (isENOENT(error)) return []
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async listJsonl(dir: string): Promise<string[]> {
|
||||
const entries = await readdir(dir)
|
||||
return entries.filter(n => n.endsWith('.jsonl'))
|
||||
}
|
||||
|
||||
private async exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
const handle = await open(path, 'r')
|
||||
await handle.close()
|
||||
return true
|
||||
} catch (error) {
|
||||
// Only ENOENT means absent. A permission/I/O error must surface, not be
|
||||
// collapsed to `false` — otherwise load() reports "not found" and collision
|
||||
// checks proceed under a false absence assumption.
|
||||
if (isENOENT(error)) return false
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default SessionPersistenceJsonl
|
||||
Reference in New Issue
Block a user