fix(session-query): harden SQLite search reconciliation

This commit is contained in:
Hypatia May
2026-07-15 12:10:24 +08:00
parent ecf90ff382
commit f88ca85ffd
40 changed files with 1315 additions and 227 deletions
@@ -12,6 +12,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l
| `append(id, events): Promise<void>` | Durably persist a batch (from the `session/flush` drain). Append-only; first event `seq` == stored next-seq after any repair; rejects non-JSON-serializable data naming the offending type. |
| `load(id): Promise<{ meta; events }>` | Reload meta + log. Preserves an interrupted (unclosed) final turn and closes it with synthetic closers — an error `tool/result` per unanswered `tool-call`, then `step/end?`+`turn/end {interrupted}` (a turn can be huge — never truncated); only a torn tail fragment is dropped. Events contiguous (`events[i].seq === i`); rejects a committed-region gap/parse error or unknown `version`. |
| `list(): Promise<SessionHeader[]>` | Lightweight listing from metadata, no full-log parse. A zero-event lazily-materialized session is absent from `list`. |
| `listSnapshots(): Promise<SessionPersistenceSnapshot[]>` | Lightweight metadata plus an opaque branded per-log revision, without loading event logs. A revision stays equal while that log is unchanged and changes after append or mutating load repair. |
## Invariants every backend must honor
@@ -24,7 +25,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l
The two first-party backends were byte-identical (or same-algorithm) for ALL of their write-path orchestration — the in-memory bookkeeping (per-id state, write-behind buffers, per-id serialization chains, per-session init promises), the `session/event` → buffer → `session/flush` drain, lazy materialization, crash-tail repair on load, the four `session/created` adoption cases (new / HMR-adopt / collision / ownerless-claim), and dispose-time quiescence. Only the STORAGE primitives differed (write bytes vs. INSERT rows).
`PersistenceCoordinator` owns that orchestration once. A first-party backend composes one (`new PersistenceCoordinator(ctx, this)`), implements the small `PersistenceBackend` hook interface, and delegates its four public service methods to the coordinator. This keeps the duplicated, correctness-heavy orchestration in a single place (it used to receive the same fixes twice).
`PersistenceCoordinator` owns that orchestration once. A first-party backend composes one (`new PersistenceCoordinator(ctx, this)`), implements the small `PersistenceBackend` hook interface, and delegates the stateful write/read methods to the coordinator. Lightweight snapshot listing remains a backend storage primitive because its revision identity is backend-owned.
The `PersistenceBackend<TornMarker>` hooks (the only seam between the coordinator and storage):
@@ -38,11 +39,11 @@ The `PersistenceBackend<TornMarker>` hooks (the only seam between the coordinato
| `list()` | List all stored metadata. |
| `close?()` | Optional lifecycle teardown (e.g. close a db handle), awaited after the dispose drain. |
The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). The public `SessionPersistence` service shape is unchanged, so a third-party backend MAY still implement the abstract service directly without the coordinator. See [the write-coordinator RFC](../../../docs/rfc/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md).
The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). A third-party backend MAY implement the abstract service directly without the coordinator, but it must also provide trustworthy lightweight snapshot revisions. See [the write-coordinator RFC](../../../docs/rfc/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md).
## Testing backends
Import `runPersistenceContract` from `tests/contract.ts` (the public-API contract) and `runCoordinatorContract` from `tests/coordinator-contract.ts` (the shared write-path orchestration: adoption, HMR, collision, dispose-drain, crash-tail repair) and call each with a fixture for your backend. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics AND the same orchestration, so a backend's own spec is left with only storage-mechanics tests (path sanitization, fsync rollback; schema version, transaction rollback) on top.
Import `runPersistenceContract` from `tests/contract.ts` (the public API, including stable/change-sensitive lightweight revisions) and `runCoordinatorContract` from `tests/coordinator-contract.ts` (the shared write-path orchestration: adoption, HMR, collision, dispose-drain, crash-tail repair) and call each with a fixture for your backend. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics AND the same orchestration, so a backend's own spec is left with only storage-mechanics tests (path sanitization, fsync rollback; schema version, transaction rollback) on top.
Three backends run these suites: an in-memory reference (in `tests/`), `dsh-session-persistence-jsonl` (append-only file log) and `dsh-session-persistence-sqlite` (`node:sqlite`, each `SessionEvent` one row `(session_id, seq, type, time, data, source_event_seqs, surface_op)`). All passing the same contract + coordinator suite is the proof that the seam is genuinely backend-agnostic — lazy materialization, crash-tail-on-load, and contiguous-seq hold identically over file bytes and over a transactional store.
@@ -22,10 +22,12 @@
],
"license": "BSD-3-Clause",
"peerDependencies": {
"@deepseek-ai/dsh-brand": "^0.0.1",
"@deepseek-ai/dsh-session": "^0.0.1",
"cordis": "^4.0.0-rc.6"
},
"devDependencies": {
"@deepseek-ai/dsh-brand": "workspace:^",
"@deepseek-ai/dsh-session": "workspace:^",
"cordis": "^4.0.0-rc.6"
}
@@ -14,7 +14,7 @@
* {@link PersistenceBackend} hook object.
*
* The abstract {@link SessionPersistence} service's public API is independent of
* this: a backend IS a `SessionPersistence` (its four public methods delegate to
* this: a backend IS a `SessionPersistence` (its write/read methods delegate to
* a coordinator it composes), so a third-party backend MAY implement the service
* directly without using the coordinator at all.
*
@@ -146,7 +146,7 @@ async function settledErrors(promises: Iterable<Promise<unknown>>): Promise<unkn
/**
* Owns the backend-agnostic session write-path orchestration. A backend
* constructs one (`new PersistenceCoordinator(ctx, this)`), implements
* {@link PersistenceBackend}, and delegates its four public service methods to
* {@link PersistenceBackend}, and delegates its write/read service methods to
* the matching coordinator methods.
*
* All per-id operations are serialized (a per-id promise chain) so concurrent
@@ -24,9 +24,19 @@
import { Context, Service } from 'cordis'
import { snapshotJsonValue } from '@deepseek-ai/dsh-session'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import type { SessionPersistenceRevision } from './revision.ts'
// Re-export the metadata vocabulary so consumers import it from the seam.
export type { SessionHeader } from '@deepseek-ai/dsh-session'
export { SessionPersistenceRevision } from './revision.ts'
/** Lightweight immutable source identity returned without loading a full log. */
export interface SessionPersistenceSnapshot {
/** Detached metadata for one materialized session. */
header: SessionHeader
/** Opaque token that changes whenever this stored log changes. */
revision: SessionPersistenceRevision
}
// The backend-agnostic write-path orchestration first-party backends compose.
export { PersistenceCoordinator } from './coordinator.ts'
@@ -156,6 +166,15 @@ export abstract class SessionPersistence extends Service {
* @returns one header per materialized session.
*/
abstract list(): Promise<SessionHeader[]>
/**
* List materialized sessions with cheap per-log change tokens.
*
* Repeated observations of an unchanged log return the same revision. A
* successful mutating {@link load} repair changes the next listed revision.
* @returns one header and opaque revision per materialized session without loading full logs.
*/
abstract listSnapshots(): Promise<SessionPersistenceSnapshot[]>
}
export default SessionPersistence
@@ -0,0 +1,15 @@
/** Opaque revision identity for lightweight persistence observations. */
import type { Branded } from '@deepseek-ai/dsh-brand'
/** Backend-owned token that changes whenever one persisted session log changes. */
export type SessionPersistenceRevision = Branded<'SessionPersistenceRevision'>
/**
* Brand a backend revision for the provider-neutral persistence contract.
* @param value - backend-owned opaque revision representation.
* @returns the same runtime string with persistence-revision identity.
*/
export function SessionPersistenceRevision(value: string): SessionPersistenceRevision {
return value as SessionPersistenceRevision
}
@@ -102,11 +102,16 @@ export function runPersistenceContract(name: string, make: () => Promise<Contrac
{ type: 'turn/start', seq: 6, time: 7, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'step/start', seq: 7, time: 8, data: { turn: 2, step: 1 } },
])
const beforeRepair = (await persistence.listSnapshots())
.find(snapshot => snapshot.header.id === m.id)?.revision
// load PRESERVES the interrupted turn's events (a turn can be huge — they
// must not be truncated) and closes the orphaned turn with synthetic
// boundary events: step/end (the step was open) then turn/end {interrupted}.
const loaded = await persistence.load(m.id)
const afterRepair = (await persistence.listSnapshots())
.find(snapshot => snapshot.header.id === m.id)?.revision
expect(afterRepair).not.toBe(beforeRepair)
expect(loaded.events.map(e => e.type)).toEqual([
'turn/start', 'user/message', 'step/start', 'assistant/message', 'step/end', 'turn/end', // turn 1
'turn/start', 'step/start', 'step/end', 'turn/end', // turn 2: real events + synthetic closers
@@ -173,18 +178,33 @@ export function runPersistenceContract(name: string, make: () => Promise<Contrac
try {
await persistence.create(meta('empty'))
expect((await persistence.list()).map(m => m.id)).not.toContain(SessionId('empty'))
expect((await persistence.listSnapshots()).map(snapshot => snapshot.header.id))
.not.toContain(SessionId('empty'))
} finally {
await dispose()
}
})
it('list() includes a session once it has events', async () => {
it('lists stable lightweight revisions that change after an append', async () => {
const { persistence, dispose } = await make()
try {
const m = meta('s2')
await persistence.create(m)
await persistence.append(m.id, oneTurnLog())
expect((await persistence.list()).map(x => x.id)).toContain(m.id)
const first = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id)
const repeated = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id)
expect(first).toBeDefined()
expect(repeated?.revision).toBe(first?.revision)
await persistence.append(m.id, [{
type: 'turn/start',
seq: 6,
time: 7,
data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } },
}])
const changed = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id)
expect(changed?.revision).not.toBe(first?.revision)
} finally {
await dispose()
}
@@ -3,8 +3,8 @@ import { Context } from 'cordis'
import SessionStore, { SessionId, isJsonValue } from '@deepseek-ai/dsh-session'
import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session'
import {
SessionPersistence, PersistenceCoordinator, assertSerializable, seedCoversPrefix,
type PersistenceBackend, type StoredPrefix,
SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, assertSerializable, seedCoversPrefix,
type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix,
} from '../src/index.ts'
import { runPersistenceContract, meta, oneTurnLog } from './contract.ts'
import { runCoordinatorContract, type CoordinatorFixture } from './coordinator-contract.ts'
@@ -109,6 +109,14 @@ class MemoryPersistence extends SessionPersistence implements PersistenceBackend
async list(): Promise<SessionHeader[]> {
return [...this.store.values()].map(e => structuredClone(e.meta))
}
async listSnapshots(): Promise<SessionPersistenceSnapshot[]> {
return [...this.store.values()].map(entry => ({
header: structuredClone(entry.meta),
revision: SessionPersistenceRevision(`events:${entry.events.length}`),
}))
}
}
// Run the shared contract against the in-memory backend.
@@ -14,6 +14,9 @@
{
"path": "../../../vendor/cordis"
},
{
"path": "../../util/brand"
},
{
"path": "../../core/session"
}