[Rust Frontend] Report reasoning tokens in chat completion usage (#54883)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -324,7 +324,20 @@ async fn chat_completion_chunk_stream(
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(ChatEvent::BlockDelta { kind, delta, .. }) => {
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
kind,
|
||||
delta,
|
||||
token_count,
|
||||
..
|
||||
}) => {
|
||||
// The count is per-delta and follows the decoder clock; it is
|
||||
// fed even when the reasoning delta itself is hidden from the
|
||||
// client (`include_reasoning=false`).
|
||||
if matches!(kind, AssistantBlockKind::Reasoning)
|
||||
&& let Some(token_count) = token_count
|
||||
{
|
||||
continuous_usage.add_reasoning_tokens(token_count);
|
||||
}
|
||||
let include_delta =
|
||||
include_reasoning || !matches!(kind, AssistantBlockKind::Reasoning);
|
||||
if include_delta {
|
||||
@@ -433,6 +446,9 @@ async fn chat_completion_chunk_stream(
|
||||
final_usage.prompt_token_count,
|
||||
final_usage.output_token_count,
|
||||
);
|
||||
// Converge the running reasoning count onto the authoritative
|
||||
// terminal value.
|
||||
continuous_usage.set_reasoning_tokens(final_usage.reasoning_tokens);
|
||||
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut()
|
||||
&& let Some(chunk) = pending_chunk.take_chunk(&envelope)
|
||||
@@ -806,16 +822,31 @@ mod tests {
|
||||
use futures::{StreamExt as _, stream};
|
||||
use serde_json::json;
|
||||
use vllm_chat::{
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, FinishReason,
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, ChatTokenUsage,
|
||||
FinishReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::{
|
||||
ApiServerOptions, ResponseOptions, StreamResponseEnvelope, block_delta_chunk,
|
||||
chat_completion_chunk_stream, final_chunk,
|
||||
ApiServerOptions, ChatCompletionStreamResponse, ResponseOptions, StreamResponseEnvelope,
|
||||
block_delta_chunk, chat_completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
/// Terminal usage with the given engine-level counts and zero reasoning tokens.
|
||||
fn done_usage(
|
||||
prompt_tokens: usize,
|
||||
output_tokens: usize,
|
||||
cached_tokens: usize,
|
||||
) -> ChatTokenUsage {
|
||||
vllm_llm::TokenUsage {
|
||||
prompt_token_count: prompt_tokens,
|
||||
output_token_count: output_tokens,
|
||||
cached_token_count: cached_tokens,
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
|
||||
Arc::new(StreamResponseEnvelope::new(
|
||||
"chatcmpl-1".to_string(),
|
||||
@@ -913,6 +944,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "hi".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -929,11 +961,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 1,
|
||||
},
|
||||
usage: done_usage(1, 1, 1),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -995,6 +1023,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1011,11 +1040,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 1, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1059,19 +1084,17 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 2, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1115,6 +1138,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1133,6 +1157,7 @@ mod tests {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1149,11 +1174,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 2, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1226,6 +1247,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1267,6 +1289,7 @@ mod tests {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1283,11 +1306,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 4,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 4, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1365,11 +1384,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 1, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1410,4 +1425,205 @@ mod tests {
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
/// Measured reasoning-token count of one chunk's continuous usage.
|
||||
fn chunk_reasoning_tokens(chunk: &ChatCompletionStreamResponse) -> Option<usize> {
|
||||
chunk
|
||||
.usage
|
||||
.as_ref()
|
||||
.map(|usage| usage.completion_tokens_details.reasoning_tokens)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunk_stream_reports_reasoning_tokens_with_continuous_usage() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![7, 8, 9].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(2),
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: None,
|
||||
token_ids: vec![20, 21],
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
// A stray text-side count must not pollute reasoning usage.
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: ChatTokenUsage {
|
||||
engine: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 3,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 2,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = chat_completion_chunk_stream(
|
||||
stream,
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
// Role, reasoning delta, text delta, finish-reason chunk, usage chunk.
|
||||
assert_eq!(chunks.len(), 5);
|
||||
// The role chunk reports reasoning tokens from 0 onwards.
|
||||
let role_usage = chunks[0].usage.as_ref().expect("role chunk usage");
|
||||
assert_eq!(role_usage.prompt_tokens, 3);
|
||||
assert_eq!(role_usage.completion_tokens, Some(0));
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[0]), Some(0));
|
||||
// The reasoning delta contributes its per-delta measured count, fed by
|
||||
// the `BlockDelta` itself rather than the sampled-clock `LogprobsDelta`.
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[1]), Some(2));
|
||||
assert_eq!(
|
||||
chunks[1].usage.as_ref().and_then(|usage| usage.completion_tokens),
|
||||
Some(2)
|
||||
);
|
||||
// The text delta's own count does not pollute reasoning usage, and the
|
||||
// terminal usage chunk converges onto the authoritative final counts.
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[2]), Some(2));
|
||||
let final_usage = chunks[4].usage.as_ref().expect("final usage chunk");
|
||||
assert_eq!(final_usage.prompt_tokens, 3);
|
||||
assert_eq!(final_usage.completion_tokens, Some(3));
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[4]), Some(2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunk_stream_reports_reasoning_tokens_when_reasoning_hidden() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(2),
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: ChatTokenUsage {
|
||||
engine: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 2,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = chat_completion_chunk_stream(
|
||||
stream,
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
// The reasoning text stays hidden...
|
||||
assert!(
|
||||
chunks
|
||||
.iter()
|
||||
.all(|chunk| chunk.choices.iter().all(|choice| choice.delta.reasoning.is_none()))
|
||||
);
|
||||
// ...but the count is still reported, from 0 on the role chunk up to
|
||||
// the converged final usage.
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[0]), Some(0));
|
||||
let final_usage = chunks.last().unwrap().usage.as_ref().expect("final usage chunk");
|
||||
assert_eq!(final_usage.completion_tokens, Some(2));
|
||||
assert_eq!(final_usage.completion_tokens_details.reasoning_tokens, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunk_stream_reports_zero_reasoning_tokens_without_reasoning() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: done_usage(1, 1, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = chat_completion_chunk_stream(
|
||||
stream,
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
// Reasoning-token details are always reported; a stream without
|
||||
// reasoning simply reports 0 on every usage payload.
|
||||
for chunk in &chunks {
|
||||
let usage = chunk.usage.as_ref().expect("usage");
|
||||
assert_eq!(usage.completion_tokens_details.reasoning_tokens, 0);
|
||||
let json = serde_json::to_value(usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::slice;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_llm::TokenUsage;
|
||||
use vllm_chat::ChatTokenUsage;
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
@@ -440,14 +440,18 @@ pub struct Usage {
|
||||
pub total_tokens: usize,
|
||||
pub completion_tokens: Option<usize>,
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
/// Reasoning-token breakdown.
|
||||
/// Always present, 0 when the configured parser has no reasoning channel.
|
||||
pub completion_tokens_details: CompletionTokenUsageInfo,
|
||||
}
|
||||
|
||||
impl Usage {
|
||||
/// Create a Usage with prompt-token cache details.
|
||||
/// Create a Usage with prompt-token cache and reasoning-token details.
|
||||
pub fn from_counts(
|
||||
prompt_tokens: usize,
|
||||
completion_tokens: usize,
|
||||
cached_tokens: Option<usize>,
|
||||
reasoning_tokens: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
prompt_tokens,
|
||||
@@ -456,14 +460,20 @@ impl Usage {
|
||||
prompt_tokens_details: cached_tokens
|
||||
.filter(|&c| c > 0)
|
||||
.map(|c| PromptTokenUsageInfo { cached_tokens: c }),
|
||||
completion_tokens_details: CompletionTokenUsageInfo { reasoning_tokens },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_token_usage(usage: TokenUsage, enable_prompt_tokens_details: bool) -> Self {
|
||||
pub fn from_token_usage(
|
||||
usage: impl Into<ChatTokenUsage>,
|
||||
enable_prompt_tokens_details: bool,
|
||||
) -> Self {
|
||||
let usage = usage.into();
|
||||
Self::from_counts(
|
||||
usage.prompt_token_count,
|
||||
usage.output_token_count,
|
||||
enable_prompt_tokens_details.then_some(usage.cached_token_count),
|
||||
usage.reasoning_tokens,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -474,8 +484,15 @@ pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: usize,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionTokenUsageInfo` class.
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct CompletionTokenUsageInfo {
|
||||
pub reasoning_tokens: usize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod usage_tests {
|
||||
use vllm_chat::ChatTokenUsage;
|
||||
use vllm_llm::TokenUsage;
|
||||
|
||||
use super::Usage;
|
||||
@@ -512,6 +529,59 @@ mod usage_tests {
|
||||
Some(3)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_includes_reasoning_token_details() {
|
||||
let usage = Usage::from_token_usage(
|
||||
ChatTokenUsage {
|
||||
engine: TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 4,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 3,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(usage.completion_tokens_details.reasoning_tokens, 3);
|
||||
let json = serde_json::to_value(&usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_serializes_zero_reasoning_tokens() {
|
||||
let usage = Usage::from_token_usage(
|
||||
ChatTokenUsage {
|
||||
engine: TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 0,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
let json = serde_json::to_value(&usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_reports_zero_reasoning_tokens_without_reasoning_parser() {
|
||||
let usage = Usage::from_token_usage(
|
||||
TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(usage.completion_tokens_details.reasoning_tokens, 0);
|
||||
let json = serde_json::to_value(&usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI completions-style logprobs.
|
||||
|
||||
@@ -12,6 +12,8 @@ use super::types::Usage;
|
||||
pub(crate) struct ContinuousUsage {
|
||||
prompt_tokens: usize,
|
||||
output_tokens: usize,
|
||||
/// Running reasoning-token count, following the decoder clock.
|
||||
reasoning_tokens: usize,
|
||||
}
|
||||
|
||||
impl ContinuousUsage {
|
||||
@@ -25,6 +27,17 @@ impl ContinuousUsage {
|
||||
self.output_tokens = self.output_tokens.saturating_add(output_tokens);
|
||||
}
|
||||
|
||||
/// Add the tokens attributed to one reasoning delta to the running count.
|
||||
pub(crate) fn add_reasoning_tokens(&mut self, token_count: usize) {
|
||||
self.reasoning_tokens = self.reasoning_tokens.saturating_add(token_count);
|
||||
}
|
||||
|
||||
/// Converge the running reasoning count onto the authoritative terminal
|
||||
/// value reported at stream end.
|
||||
pub(crate) fn set_reasoning_tokens(&mut self, reasoning_tokens: usize) {
|
||||
self.reasoning_tokens = reasoning_tokens;
|
||||
}
|
||||
|
||||
/// Replace the running counts with the final counts reported by generation.
|
||||
pub(crate) fn set_final_counts(&mut self, prompt_tokens: usize, output_tokens: usize) {
|
||||
self.prompt_tokens = prompt_tokens;
|
||||
@@ -33,6 +46,11 @@ impl ContinuousUsage {
|
||||
|
||||
/// Build a streaming usage snapshot without prompt cache details.
|
||||
pub(crate) fn to_usage(&self) -> Usage {
|
||||
Usage::from_counts(self.prompt_tokens, self.output_tokens, None)
|
||||
Usage::from_counts(
|
||||
self.prompt_tokens,
|
||||
self.output_tokens,
|
||||
None,
|
||||
self.reasoning_tokens,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user