fix(fixture): avoid duplicated token estimator shape
This commit is contained in:
@@ -838,26 +838,19 @@ const ROLE_OVERHEAD = 4
|
|||||||
|
|
||||||
/** Price fixture content with token-meter's fixed-density heuristic. */
|
/** Price fixture content with token-meter's fixed-density heuristic. */
|
||||||
function estimateFixtureContent(blocks: readonly ContentBlock[]): number {
|
function estimateFixtureContent(blocks: readonly ContentBlock[]): number {
|
||||||
let tokens = 0
|
const densityPrice = (value: string): number => Math.ceil(value.length / CHARS_PER_TOKEN)
|
||||||
for (const block of blocks) {
|
return blocks.reduce((tokens, block) => {
|
||||||
switch (block.type) {
|
if (block.type === 'text' || block.type === 'reasoning') {
|
||||||
case 'text':
|
return tokens + densityPrice(block.text) + BLOCK_OVERHEAD
|
||||||
case 'reasoning':
|
|
||||||
tokens += Math.ceil(block.text.length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
|
|
||||||
break
|
|
||||||
case 'tool-call':
|
|
||||||
tokens += Math.ceil(block.name.length / CHARS_PER_TOKEN)
|
|
||||||
+ Math.ceil(block.arguments.length / CHARS_PER_TOKEN)
|
|
||||||
+ BLOCK_OVERHEAD
|
|
||||||
break
|
|
||||||
case 'tool-result':
|
|
||||||
tokens += estimateFixtureContent(block.content) + BLOCK_OVERHEAD
|
|
||||||
break
|
|
||||||
default:
|
|
||||||
tokens += BLOCK_OVERHEAD + Math.ceil(JSON.stringify(block).length / CHARS_PER_TOKEN)
|
|
||||||
}
|
}
|
||||||
}
|
if (block.type === 'tool-call') {
|
||||||
return tokens
|
return tokens + densityPrice(block.name) + densityPrice(block.arguments) + BLOCK_OVERHEAD
|
||||||
|
}
|
||||||
|
if (block.type === 'tool-result') {
|
||||||
|
return tokens + estimateFixtureContent(block.content) + BLOCK_OVERHEAD
|
||||||
|
}
|
||||||
|
return tokens + densityPrice(JSON.stringify(block)) + BLOCK_OVERHEAD
|
||||||
|
}, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Fixture parallel of token-meter's heuristic context-composition projection. */
|
/** Fixture parallel of token-meter's heuristic context-composition projection. */
|
||||||
|
|||||||
Reference in New Issue
Block a user