Pine/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat(telemetry): add feedback-gated OTEL modes

Browse Source
This commit is contained in:
Turtle
2026-08-05 12:43:35 +08:00
parent b8d51704f3
commit c836fcd416
41 changed files with 635 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/telemetry/session-telemetry-otel/tests/loader-composition.e2e.ts
+74 -19
View File
@@ -40,6 +40,11 @@ interface OtlpCapture {
}[]
}
interface FixtureOutput {
captures: OtlpCapture[]
logContent: string
}
async function jsonlFiles(dir: string): Promise<string[]> {
const entries = await readdir(dir, { withFileTypes: true })
const paths = await Promise.all(entries.map(async (entry) => {
@@ -50,10 +55,29 @@ async function jsonlFiles(dir: string): Promise<string[]> {
return paths.flat()
}
async function readFixtureOutput(cwd: string): Promise<FixtureOutput> {
const captures = JSON.parse(await readFile(join(cwd, 'otlp-captures.json'), 'utf8')) as OtlpCapture[]
const logs = await jsonlFiles(join(cwd, '.sessions'))
expect(logs).toHaveLength(1)
return { captures, logContent: await readFile(logs[0] as string, 'utf8') }
}
function allRecords(captures: OtlpCapture[]) {
return captures.flatMap(capture => capture.resourceLogs.flatMap(resource =>
resource.scopeLogs.flatMap(scoped => scoped.logRecords.map(record => ({ scope: scoped.scope.name, record })))))
}
function eventTypes(captures: OtlpCapture[]): string[] {
return allRecords(captures).flatMap(({ record }) =>
record.attributes?.flatMap(attribute =>
attribute.key === 'event.type' && typeof attribute.value['stringValue'] === 'string'
? [attribute.value['stringValue']]
: []) ?? [])
}
describe('session-telemetry-otel through a real headless cordis.yml', () => {
it('exports redacted ledger records to the collector while the canonical log keeps the secret', async () => {
let captures: OtlpCapture[] = []
let logContent = ''
let output!: FixtureOutput
const { stderr } = await runLoaderSmoke({
label: 'session-telemetry-otel loader smoke',
tempDirPrefix: 'telemetry-otel-e2e-',
@@ -61,39 +85,70 @@ describe('session-telemetry-otel through a real headless cordis.yml', () => {
libBinScript: driver,
configPath,
tsconfigPath: repoTsconfig,
inspect: async (cwd) => {
captures = JSON.parse(await readFile(join(cwd, 'otlp-captures.json'), 'utf8')) as OtlpCapture[]
const logs = await jsonlFiles(join(cwd, '.sessions'))
expect(logs).toHaveLength(1)
logContent = await readFile(logs[0] as string, 'utf8')
},
inspect: async (cwd) => { output = await readFixtureOutput(cwd) },
})
expect(stderr).not.toContain('UNHANDLED')
const records = captures.flatMap(capture => capture.resourceLogs.flatMap(resource =>
resource.scopeLogs.flatMap(scoped => scoped.logRecords.map(record => ({ scope: scoped.scope.name, record })))))
const records = allRecords(output.captures)
expect(records.length).toBeGreaterThan(0)
const eventTypes = records.flatMap(({ record }) =>
record.attributes?.flatMap(attribute =>
attribute.key === 'event.type' && typeof attribute.value['stringValue'] === 'string'
? [attribute.value['stringValue']]
: []) ?? [])
const types = eventTypes(output.captures)
for (const expected of ['turn/start', 'user/message', 'tool/call', 'tool/result', 'assistant/message', 'turn/end']) {
expect(eventTypes, expected).toContain(expected)
expect(types, expected).toContain(expected)
}
expect(records.some(({ scope }) => scope.endsWith('/ops'))).toBe(true)
// The deployment-mounted rule on the wire: the fixture credential never
// leaves the process, its surrounding prose does, and the placeholder
// marks the spot — the seam itself ships no rules.
const wire = JSON.stringify(captures)
const wire = JSON.stringify(output.captures)
expect(wire).not.toContain(FIXTURE_SECRET)
expect(wire).toContain(FIXTURE_PLACEHOLDER)
expect(wire).toContain('prove telemetry with key')
// The canonical session log is never rewritten.
expect(logContent).toContain(FIXTURE_SECRET)
expect(logContent).not.toContain(FIXTURE_PLACEHOLDER)
expect(output.logContent).toContain(FIXTURE_SECRET)
expect(output.logContent).not.toContain(FIXTURE_PLACEHOLDER)
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('exports only prefixes ending in feedback under feedback-only mode', async () => {
let output!: FixtureOutput
const { stderr } = await runLoaderSmoke({
label: 'session-telemetry-otel feedback-only loader smoke',
tempDirPrefix: 'telemetry-otel-feedback-e2e-',
binScript: driver,
libBinScript: driver,
configPath,
tsconfigPath: repoTsconfig,
env: { DSH_TELEMETRY_E2E_MODE: 'FEEDBACK_ONLY' },
inspect: async (cwd) => { output = await readFixtureOutput(cwd) },
})
expect(stderr).not.toContain('UNHANDLED')
const wire = JSON.stringify(output.captures)
expect(eventTypes(output.captures)).toContain('feedback/record')
expect(wire).toContain('fixture feedback')
expect(wire).toContain('prove telemetry with key')
expect(wire).not.toContain('post-feedback private suffix')
expect(output.logContent).toContain('post-feedback private suffix')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('keeps disabled feedback local and prints the stable warning', async () => {
let output!: FixtureOutput
const { stdout } = await runLoaderSmoke({
label: 'session-telemetry-otel disabled loader smoke',
tempDirPrefix: 'telemetry-otel-disabled-e2e-',
binScript: driver,
libBinScript: driver,
configPath,
tsconfigPath: repoTsconfig,
env: { DSH_TELEMETRY_E2E_MODE: 'DISABLED' },
inspect: async (cwd) => { output = await readFixtureOutput(cwd) },
})
expect(output.captures).toEqual([])
expect(output.logContent).toContain('fixture feedback')
expect(stdout.match(/session telemetry is DISABLED; nothing will be shared and this feedback remains local/)?.[0])
.toMatchInlineSnapshot('"session telemetry is DISABLED; nothing will be shared and this feedback remains local"')
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
})
packages/telemetry/session-telemetry-otel/tests/otel.spec.ts
+89 -1
View File
@@ -5,12 +5,13 @@
* for the default-exported Service class.
*/
import { afterEach, describe, expect, it } from 'vitest'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { createServer, type Server } from 'node:http'
import { once } from 'node:events'
import { gunzipSync } from 'node:zlib'
import { Context } from 'cordis'
import Loader from '@cordisjs/plugin-loader'
import { recordFeedback } from '@deepseek-ai/dsh-command-feedback'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import TelemetryOtel, { Config } from '../src/index.ts'
@@ -30,6 +31,7 @@ interface OtlpLogsRequest {
severityNumber: number
severityText: string
attributes?: { key: string; value: Record<string, unknown> }[]
body?: unknown
}[]
}[]
}[]
@@ -88,6 +90,14 @@ function allRecords(captures: Capture[]) {
s.logRecords.map(record => ({ scope: s.scope.name, record })))))
}
function eventTypes(captures: Capture[]): string[] {
return allRecords(captures).flatMap(({ record }) =>
record.attributes?.flatMap(attribute =>
attribute.key === 'event.type' && typeof attribute.value['stringValue'] === 'string'
? [attribute.value['stringValue']]
: []) ?? [])
}
describe('TelemetryOtel wire', () => {
it('ships session records and the ops shutdown marker through the real SDK pipeline', async () => {
const { url, captures } = await mockCollector()
@@ -195,6 +205,82 @@ describe('TelemetryOtel wire', () => {
r.record.attributes?.some(a => a.key === 'event.type' && a.value.stringValue === 'turn/start'))
expect(start?.record.severityNumber).toBe(13)
})
it('holds each session suffix until the next feedback event', async () => {
const { url, captures } = await mockCollector()
const ctx = new Context()
await ctx.plugin(SessionStore)
const fiber = await ctx.plugin(TelemetryOtel, {
mode: 'FEEDBACK_ONLY',
exporter: { url },
})
const session = ctx.sessions.create(SessionId('feedback-only'), { meta: {} })
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
recordFeedback(session, 'first report')
session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
recordFeedback(session, 'second report')
session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } })
await fiber.dispose()
const types = allRecords(captures).flatMap(({ record }) =>
record.attributes?.flatMap(attribute =>
attribute.key === 'event.type' ? [attribute.value.stringValue] : []) ?? [])
expect(types).toEqual(['turn/start', 'feedback/record', 'turn/end', 'feedback/record'])
expect(JSON.stringify(captures)).toContain('first report')
expect(JSON.stringify(captures)).toContain('second report')
expect(allRecords(captures).some(({ scope }) => scope.endsWith('/ops'))).toBe(false)
})
it('sends no request when feedback-only mode ends without feedback', async () => {
const { url, captures } = await mockCollector()
const ctx = new Context()
await ctx.plugin(SessionStore)
const fiber = await ctx.plugin(TelemetryOtel, {
mode: 'FEEDBACK_ONLY',
exporter: { url },
})
const session = ctx.sessions.create(SessionId('no-feedback'), { meta: {} })
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
await fiber.dispose()
expect(captures).toEqual([])
})
it('boots disabled without exporter config and warns when feedback stays local', async () => {
const ctx = new Context()
await ctx.plugin(SessionStore)
const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {})
const fiber = await ctx.plugin(TelemetryOtel, { mode: 'DISABLED' })
const session = ctx.sessions.create(SessionId('disabled'), { meta: {} })
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
recordFeedback(session, 'local report')
expect(warn).toHaveBeenCalledWith(
'session telemetry is DISABLED; nothing will be shared and this feedback remains local',
)
ctx.telemetry.emit({
channel: 'ledger',
time: 0,
severity: 'info',
attributes: {},
body: null,
})
await ctx.telemetry.shutdown()
await fiber.dispose()
recordFeedback(session, 'after disposal')
expect(warn).toHaveBeenCalledTimes(1)
})
it('defaults direct construction to full delivery', async () => {
const { url, captures } = await mockCollector()
const ctx = new Context()
await ctx.plugin(SessionStore)
new TelemetryOtel(ctx, { exporter: { url } })
const session = ctx.sessions.create(SessionId('direct-default'), { meta: {} })
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
await ctx.fiber.dispose()
expect(eventTypes(captures)).toContain('turn/start')
})
})
describe('TelemetryOtel config fails loud', () => {
@@ -203,6 +289,8 @@ describe('TelemetryOtel config fails loud', () => {
[{ exporter: { url: '' } }, /exporter\.url is required/],
[{ exporter: { url: 'not a url' } }, /not a valid URL/],
[{ exporter: { url: 'ftp://collector' } }, /must be http\(s\)/],
[{ mode: 'FEEDBACK_ONLY' }, /exporter\.url is required/],
[{ mode: 'INVALID' }, /INVALID/],
// The SDK accepts a non-positive batch size but its shutdown drain then
// splices empty batches forever — dispose would hang, so reject at load.
[{ exporter: { url: 'http://c/v1/logs' }, processor: { maxExportBatchSize: 0 } }, /maxExportBatchSize/],