Pine/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat(session): opt-in packed chunk rows in the JSONL log

Browse Source 
Providers stream token-sized deltas, so a session log stores hundreds of
near-identical assistant/chunk lines whose JSON envelopes dwarf their
payloads (~56x measured on a real DeepSeek session, 73% of file bytes).

Add a lossless storage codec to dsh-session: packChunkRuns() folds each
run of >=3 consecutive same-block delta chunks into one storage row --
text-chunks / reasoning-chunks / tool-call-chunks, bare slash-less tags
like the header line's 'session' so rows cannot be confused with session
events -- and decodeStorageRecord() expands rows back to the exact
original events (seq0/time0 + dt gap array reconstruct every member's
seq/time; tool-call rows carry the run-constant id/name). The encoder
whitelists exact shapes and stores anything unrecognized verbatim; the
decoder validates row-tagged values and fails loud on malformation.

The JSONL backend gains a packChunks config (default false). Writing
packs only when enabled -- default-off output stays byte-identical to
the previous layout, so snapshot goldens are untouched. Reading is
layout-blind: scanLog always decodes rows and now checks seq contiguity
with a cursor instead of the line index, so packed, unpacked, and mixed
files all load identically. Fixture readers (llm-replay parseSessionLog,
acp-snapshot normalizeSessionLog) share the codec; the normalizer zeroes
a row's time0/dt exactly like an event's time. The two demo bundles
plumb packChunks from cordis.yml to the backend.

Measured on a real coding session: 105 KB -> 42 KB (-60%), 475 lines ->
74, with reasoning/tool-call heavy sessions saving the most. Covered by
example + fast-check round-trip codec tests, backend packed/mixed/torn-
tail specs, and an end-to-end demo run loading a packed log through a
default-config backend.
This commit is contained in:
kingwl
2026-07-15 21:26:36 +08:00
parent b045b553a9
commit 7ef21239ca
21 changed files with 816 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/session-persistence/session-persistence-jsonl/src/format.ts
+49 -25
View File
@@ -10,7 +10,8 @@
import { createHash } from 'node:crypto'
import { join } from 'node:path'
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session'
import type { SessionEvent, SessionHeader, SessionId, StorageRecord } from '@deepseek-ai/dsh-session'
/**
* The first line of a session's `.jsonl` file: the immutable
@@ -126,17 +127,26 @@ export function logPath(root: string, cwd: string | undefined, id: SessionId): s
}
/**
* Serialize one event as a JSONL line (no trailing newline).
* @param event - the event to serialize verbatim.
* @returns the event's single-line JSON text; the writer adds the newline.
* Serialize an event batch as JSONL lines (no trailing newline). With
* `packChunks` on, delta-chunk runs pack into `text-chunks` /
* `reasoning-chunks` / `tool-call-chunks` storage rows; off writes one event
* per line, byte-identical to the pre-packing layout. Reading is layout-blind
* either way ({@link scanLog} always decodes rows), so the switch only shapes
* NEW bytes.
* @param events - the batch to serialize, in log order.
* @param packChunks - whether to pack delta runs into storage rows.
* @returns the batch's JSONL text; the writer adds the final newline.
*/
export function eventLine(event: SessionEvent): string {
return JSON.stringify(event)
export function eventLines(events: readonly SessionEvent[], packChunks: boolean): string {
const records: readonly StorageRecord[] = packChunks ? packChunkRuns(events) : events
return records.map(record => JSON.stringify(record)).join('\n')
}
/**
* Parse a JSONL log buffer into its preserved event prefix (the header is line
* 0). Fully written events in an interrupted final turn remain part of the
* 0). Event lines pass through verbatim; packed chunk rows expand back into
* their events, so callers see one contiguous event list regardless of layout.
* Fully written events in an interrupted final turn remain part of the
* prefix. The first unparsable record or seq gap after the last `turn/end`
* marks a tolerated torn tail; the same hole in the committed region rejects.
*
@@ -175,46 +185,60 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
}
const headerLine = parsedHeader
// Parse every complete record first so the last valid `turn/end` determines
// whether an earlier hole is committed corruption or an uncommitted tail.
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
// Parse and decode every complete line first so the last valid `turn/end`
// determines whether an earlier hole is committed corruption or an
// uncommitted tail. One line yields one event, or a whole run for a packed
// chunk row; a row-tagged line that fails row validation is a hole, exactly
// like unparsable JSON.
interface Parsed { ok: boolean; events?: SessionEvent[]; endByte: number }
const parsed: Parsed[] = eventEntries.map((entry) => {
try {
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
return { ok: true, events: decodeStorageRecord(JSON.parse(entry.text)), endByte: entry.endByte }
} catch {
return { ok: false, endByte: entry.endByte }
}
})
// The last index (into eventEntries) that is a valid `turn/end` — the last
// fully-committed boundary (the loop flushes only at turn/end).
// The last index (into eventEntries) that ends in a valid `turn/end` — the
// last fully-committed boundary (the loop flushes only at turn/end). A packed
// row never stores a turn/end, so only single-event lines can match.
let lastTurnEnd = -1
for (let i = parsed.length - 1; i >= 0; i--) {
const p = parsed[i]
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
if (p?.ok && p.events?.some(e => e.type === 'turn/end')) { lastTurnEnd = i; break }
}
// Preserve the contiguous prefix, including a complete interrupted turn;
// holes through the last committed boundary throw, while later holes stop.
// Contiguity is a cursor over seqs (not the line index): a packed row
// advances the cursor by its whole run.
const preserved: SessionEvent[] = []
for (let i = 0; i < parsed.length; i++) {
let lastPreservedLine = -1
scan: for (let i = 0; i < parsed.length; i++) {
const p = parsed[i]
if (!p?.ok || p.event === undefined) {
if (!p?.ok || p.events === undefined) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
break // torn tail fragment after the last turn/end — stop, tolerate
}
if (p.event.seq !== i) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
break // gap after the last turn/end — torn tail, stop
for (const event of p.events) {
if (event.seq !== preserved.length) {
if (i <= lastTurnEnd) {
throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${preserved.length}, got ${event.seq})`)
}
break scan // gap after the last turn/end — torn tail, stop
}
preserved.push(event)
}
preserved.push(p.event)
lastPreservedLine = i
}
// committedBytes = end of the last PRESERVED line (header if none): the next
// append truncates any torn bytes past this point before writing the
// synthetic closers + new events.
const lastPreserved = parsed[preserved.length - 1]
const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte
// committedBytes = end of the last FULLY preserved line (header if none): the
// next append truncates any torn bytes past this point before writing the
// synthetic closers + new events. A line is preserved whole or not at all —
// a mid-row seq gap discards the whole row, keeping the truncation offset on
// a line boundary.
const lastPreserved = parsed[lastPreservedLine]
const committedBytes = lastPreserved !== undefined ? lastPreserved.endByte : headerEntry.endByte
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
}
packages/session-persistence/session-persistence-jsonl/src/index.ts
+18 -4
View File
@@ -16,10 +16,10 @@ import {
} from '@deepseek-ai/dsh-session-persistence'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import {
encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
encodeSegment, eventLines, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
} from './format.ts'
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */
export interface Config {
/**
* Root directory for all session files. Required (no default): a default of
@@ -27,6 +27,15 @@ export interface Config {
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
*/
root: string
/**
* Write runs of consecutive `assistant/chunk` delta events as packed
* `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless,
* ~60% smaller logs measured on a real session). Off by default while
* snapshot fixtures stay in the one-event-per-line layout: recording with
* packing on rewrites every golden `session.jsonl`. READING packed rows is
* unconditional — a log's layout never depends on this switch.
*/
packChunks?: boolean
}
/** Whether a filesystem error means absence; every non-ENOENT failure must surface. */
@@ -44,6 +53,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
static Config: z<Config> = z.object({
root: z.string().required(),
packChunks: z.boolean().default(false),
})
/**
@@ -54,12 +64,16 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
override readonly name = 'session-persistence-jsonl'
private root: string
private packChunks: boolean
private coordinator: PersistenceCoordinator<number>
constructor(ctx: Context, public config: Config) {
super(ctx)
// Resolve once so later process.cwd() changes cannot split one backend across roots.
this.root = resolve(config.root)
// schemastery (static Config) applied the default before construction;
// the cast records that runtime fact for exactOptionalPropertyTypes.
this.packChunks = (config as Required<Config>).packChunks
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
}
@@ -168,7 +182,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
}
const header = JSON.stringify(toHeaderLine(meta))
const body = events.map(eventLine).join('\n')
const body = eventLines(events, this.packChunks)
const content = header + '\n' + body + '\n'
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
@@ -223,7 +237,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
try {
const { size: before } = await handle.stat()
try {
await handle.writeFile(events.map(eventLine).join('\n') + '\n')
await handle.writeFile(eventLines(events, this.packChunks) + '\n')
await handle.sync()
} catch (error) {
// Roll back whatever bytes landed so a retry starts from a clean EOF.