Pine/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

Merge master into codex/jsonl-storage-identity

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-23 20:12:26 +08:00
parent a1f3215c5a ebc415303d
commit a5cb16cc0e
1289 changed files with 84682 additions and 7686 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/session-persistence/session-persistence-jsonl/src/format.ts
+49 -25
View File
@@ -10,7 +10,8 @@
import { createHash } from 'node:crypto'
import { join } from 'node:path'
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session'
import type { SessionEvent, SessionHeader, SessionId, StorageRecord } from '@deepseek-ai/dsh-session'
/** Physical encoding selected for JSONL session artifacts. */
export type JsonlCompression = 'zstd' | 'none'
@@ -151,17 +152,26 @@ export function logPath(
}
/**
* Serialize one event as a JSONL line (no trailing newline).
* @param event - the event to serialize verbatim.
* @returns the event's single-line JSON text; the writer adds the newline.
* Serialize an event batch as JSONL lines (no trailing newline). With
* `packChunks` on, delta-chunk runs pack into `text-chunks` /
* `reasoning-chunks` / `tool-call-chunks` storage rows; off writes one event
* per line, byte-identical to the pre-packing layout. Reading is layout-blind
* either way ({@link scanLog} always decodes rows), so the switch only shapes
* NEW bytes.
* @param events - the batch to serialize, in log order.
* @param packChunks - whether to pack delta runs into storage rows.
* @returns the batch's JSONL text; the writer adds the final newline.
*/
export function eventLine(event: SessionEvent): string {
return JSON.stringify(event)
export function eventLines(events: readonly SessionEvent[], packChunks: boolean): string {
const records: readonly StorageRecord[] = packChunks ? packChunkRuns(events) : events
return records.map(record => JSON.stringify(record)).join('\n')
}
/**
* Parse a JSONL log buffer into its preserved event prefix (the header is line
* 0). Fully written events in an interrupted final turn remain part of the
* 0). Event lines pass through verbatim; packed chunk rows expand back into
* their events, so callers see one contiguous event list regardless of layout.
* Fully written events in an interrupted final turn remain part of the
* prefix. The first unparsable record or seq gap after the last `turn/end`
* marks a tolerated torn tail; the same hole in the committed region rejects.
*
@@ -200,46 +210,60 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
}
const headerLine = parsedHeader
// Parse every complete record first so the last valid `turn/end` determines
// whether an earlier hole is committed corruption or an uncommitted tail.
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
// Parse and decode every complete line first so the last valid `turn/end`
// determines whether an earlier hole is committed corruption or an
// uncommitted tail. One line yields one event, or a whole run for a packed
// chunk row; a row-tagged line that fails row validation is a hole, exactly
// like unparsable JSON.
interface Parsed { ok: boolean; events?: SessionEvent[]; endByte: number }
const parsed: Parsed[] = eventEntries.map((entry) => {
try {
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
return { ok: true, events: decodeStorageRecord(JSON.parse(entry.text)), endByte: entry.endByte }
} catch {
return { ok: false, endByte: entry.endByte }
}
})
// The last index (into eventEntries) that is a valid `turn/end` — the last
// fully-committed boundary (the loop flushes only at turn/end).
// The last index (into eventEntries) that ends in a valid `turn/end` — the
// last fully-committed boundary (the loop flushes only at turn/end). A packed
// row never stores a turn/end, so only single-event lines can match.
let lastTurnEnd = -1
for (let i = parsed.length - 1; i >= 0; i--) {
const p = parsed[i]
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
if (p?.ok && p.events?.some(e => e.type === 'turn/end')) { lastTurnEnd = i; break }
}
// Preserve the contiguous prefix, including a complete interrupted turn;
// holes through the last committed boundary throw, while later holes stop.
// Contiguity is a cursor over seqs (not the line index): a packed row
// advances the cursor by its whole run.
const preserved: SessionEvent[] = []
for (let i = 0; i < parsed.length; i++) {
let lastPreservedLine = -1
scan: for (let i = 0; i < parsed.length; i++) {
const p = parsed[i]
if (!p?.ok || p.event === undefined) {
if (!p?.ok || p.events === undefined) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
break // torn tail fragment after the last turn/end — stop, tolerate
}
if (p.event.seq !== i) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
break // gap after the last turn/end — torn tail, stop
for (const event of p.events) {
if (event.seq !== preserved.length) {
if (i <= lastTurnEnd) {
throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${preserved.length}, got ${event.seq})`)
}
break scan // gap after the last turn/end — torn tail, stop
}
preserved.push(event)
}
preserved.push(p.event)
lastPreservedLine = i
}
// committedBytes = end of the last PRESERVED line (header if none): the next
// append truncates any torn bytes past this point before writing the
// synthetic closers + new events.
const lastPreserved = parsed[preserved.length - 1]
const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte
// committedBytes = end of the last FULLY preserved line (header if none): the
// next append truncates any torn bytes past this point before writing the
// synthetic closers + new events. A line is preserved whole or not at all —
// a mid-row seq gap discards the whole row, keeping the truncation offset on
// a line boundary.
const lastPreserved = parsed[lastPreservedLine]
const committedBytes = lastPreserved !== undefined ? lastPreserved.endByte : headerEntry.endByte
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
}
packages/session-persistence/session-persistence-jsonl/src/index.ts
+18 -4
View File
@@ -18,7 +18,7 @@ import {
} from '@deepseek-ai/dsh-session-persistence'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import {
encodeSegment, eventLine, logPath, logSuffix, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
encodeSegment, eventLines, logPath, logSuffix, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
type JsonlCompression,
} from './format.ts'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from './zstd.ts'
@@ -34,7 +34,7 @@ export const JsonlCompressionSchema: z<JsonlCompression> = z.union([
z.const('none'),
]).default(DEFAULT_COMPRESSION)
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */
export interface Config {
/**
* Root directory for all session files. Required (no default): a default of
@@ -44,6 +44,15 @@ export interface Config {
* first materialization.
*/
root: string
/**
* Write runs of consecutive `assistant/chunk` delta events as packed
* `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless,
* ~60% smaller logs measured on a real session). Off by default while
* snapshot fixtures stay in the one-event-per-line layout: recording with
* packing on rewrites every golden `session.jsonl`. READING packed rows is
* unconditional — a log's layout never depends on this switch.
*/
packChunks?: boolean
/** Physical encoding; defaults to checksummed Zstandard frames. */
compression?: JsonlCompression
}
@@ -70,6 +79,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
static Config: z<Config> = z.object({
root: z.string().required(),
packChunks: z.boolean().default(false),
compression: JsonlCompressionSchema,
})
@@ -81,6 +91,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
override readonly name = 'session-persistence-jsonl'
private root: string
private packChunks: boolean
private compression: JsonlCompression
private coordinator: PersistenceCoordinator<JsonlTornMarker>
private rootEncodingCheck: Promise<void> | undefined
@@ -89,6 +100,9 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
super(ctx)
// Resolve once so later process.cwd() changes cannot split one backend across roots.
this.root = resolve(config.root)
// schemastery (static Config) applied the default before construction;
// the cast records that runtime fact for exactOptionalPropertyTypes.
this.packChunks = (config as Required<Config>).packChunks
this.compression = config.compression ?? DEFAULT_COMPRESSION
this.assertUsableRoot()
this.coordinator = new PersistenceCoordinator<JsonlTornMarker>(this.ctx, this)
@@ -357,7 +371,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Encode the header and first batch without combining their frame boundaries. */
private async encodeMaterialization(meta: SessionHeader, events: readonly SessionEvent[]): Promise<Buffer | string> {
const header = JSON.stringify(toHeaderLine(meta)) + '\n'
const body = events.map(eventLine).join('\n') + '\n'
const body = eventLines(events, this.packChunks) + '\n'
if (this.compression === 'none') return header + body
const headerFrame = await compressZstdFrame(header)
const eventFrame = await compressZstdFrame(body)
@@ -366,7 +380,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Encode one durable append batch in the configured physical representation. */
private async encodeEventBatch(events: readonly SessionEvent[]): Promise<Buffer | string> {
const body = events.map(eventLine).join('\n') + '\n'
const body = eventLines(events, this.packChunks) + '\n'
return this.compression === 'zstd' ? compressZstdFrame(body) : body
}