[Rust Frontend] Report reasoning tokens in chat completion usage (#54883)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-03 16:47:24 -07:00
committed by GitHub
parent 579aef4e8d
commit d6bce42983
15 changed files with 735 additions and 88 deletions
+34 -1
View File
@@ -123,6 +123,37 @@ impl [AssistantContentBlock] {
}
}
/// Chat-level token usage for one completed request.
///
/// Extends the engine-level [`TokenUsage`] with reasoning attribution measured
/// by the chat output pipeline. [`TokenUsage`] itself stays engine-level and
/// is not touched by parsing.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ChatTokenUsage {
/// Engine-level token usage.
pub engine: TokenUsage,
/// Number of generated tokens attributed to reasoning; 0 when the
/// configured parser has no reasoning channel.
pub reasoning_tokens: usize,
}
impl From<TokenUsage> for ChatTokenUsage {
fn from(engine: TokenUsage) -> Self {
Self {
engine,
reasoning_tokens: 0,
}
}
}
impl Deref for ChatTokenUsage {
type Target = TokenUsage;
fn deref(&self) -> &Self::Target {
&self.engine
}
}
/// Final structured assistant message assembled from the event stream.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct AssistantMessage {
@@ -172,6 +203,8 @@ pub enum ChatEvent {
index: usize,
kind: AssistantBlockKind,
delta: String,
/// Number of generated tokens attributed to this delta, when available.
token_count: Option<usize>,
},
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
LogprobsDelta {
@@ -201,7 +234,7 @@ pub enum ChatEvent {
/// metadata.
Done {
message: AssistantMessage,
usage: TokenUsage,
usage: ChatTokenUsage,
finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
kv_transfer_params: Option<serde_json::Value>,
+1 -1
View File
@@ -18,7 +18,7 @@ pub use backend::{
pub use error::{Error, Result};
pub use event::{
AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantMessageExt,
AssistantToolCall, ChatEvent,
AssistantToolCall, ChatEvent, ChatTokenUsage,
};
use futures::{StreamExt, TryStreamExt as _};
pub use llm_multimodal::MediaContentPart;
+214 -14
View File
@@ -17,7 +17,7 @@ use vllm_text::output::{DecodedText, DecodedTextEvent, SampledDelta};
use crate::Result;
use crate::error::Error;
use crate::event::AssistantBlockKind;
use crate::event::{AssistantBlockKind, ChatTokenUsage};
use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id};
/// Per-stream unified parsing state.
@@ -31,6 +31,10 @@ struct UnifiedParserState {
/// Supported parsers currently emit at most one active tool call at a time.
/// Change this to an indexed map if a model needs interleaved calls later.
open_call_index: Option<usize>,
/// Running count of tokens attributed to reasoning. Kept solely to
/// produce the final count on `Done`; per-delta counts are carried by
/// each `TextDelta` event itself.
reasoning_tokens: usize,
}
impl UnifiedParserState {
@@ -40,6 +44,7 @@ impl UnifiedParserState {
parser,
parser_failed: false,
open_call_index: None,
reasoning_tokens: 0,
}
}
@@ -69,10 +74,10 @@ impl UnifiedParserState {
return Ok(text_event(AssistantBlockKind::Text, delta.text).into_iter().collect());
}
let mut output = UnifiedParserOutput::default();
// The parser consumes the delta; keep its text for the fallback path
// that re-emits it as plain text.
let fallback_text = delta.text.clone();
let mut output = UnifiedParserOutput::default();
match self.parser.parse_into(delta, &mut output) {
Ok(()) => {
let mut events = Vec::new();
@@ -138,9 +143,13 @@ impl UnifiedParserState {
self.open_call_index = None;
push_text_delta(events, AssistantBlockKind::Text, delta);
}
UnifiedParserEvent::Reasoning(delta) => {
UnifiedParserEvent::Reasoning(piece) => {
self.open_call_index = None;
push_text_delta(events, AssistantBlockKind::Reasoning, delta.text);
// By construction this counts exactly the tokens emitted to
// the client as reasoning, so the parser-failure fallback
// needs no special handling.
self.reasoning_tokens += piece.attributions.len();
push_piece_delta(events, AssistantBlockKind::Reasoning, piece);
}
UnifiedParserEvent::ToolCall(item) => {
self.process_tool_item(item, events)?;
@@ -200,12 +209,17 @@ impl UnifiedParserState {
}
}
/// Build one plain text event if `delta` is non-empty.
/// Build one plain text event with no measured tokens, if `delta` is
/// non-empty.
fn text_event(kind: AssistantBlockKind, delta: String) -> Option<AssistantEvent> {
if delta.is_empty() {
return None;
}
Some(AssistantEvent::TextDelta { kind, delta })
Some(AssistantEvent::TextDelta {
kind,
delta,
token_count: None,
})
}
/// Push one plain text delta if it is non-empty.
@@ -215,6 +229,22 @@ fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, d
}
}
/// Push one attributed delta, counting its tokens, if its text is non-empty.
fn push_piece_delta(
events: &mut Vec<AssistantEvent>,
kind: AssistantBlockKind,
piece: DecodedText,
) {
if piece.text.is_empty() {
return;
}
events.push(AssistantEvent::TextDelta {
kind,
delta: piece.text,
token_count: Some(piece.attributions.len()),
});
}
/// Wrap one decoded-text stream into the internal unified assistant stream.
#[try_stream]
pub(crate) async fn unified_event_stream(
@@ -263,7 +293,10 @@ pub(crate) async fn unified_event_stream(
y.yield_ok(next).await;
}
y.yield_ok(AssistantEvent::Done {
usage: finished.usage,
usage: ChatTokenUsage {
engine: finished.usage,
reasoning_tokens: state.reasoning_tokens,
},
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
ec_transfer_params: finished.ec_transfer_params,
@@ -285,10 +318,12 @@ mod tests {
use vllm_parser::reasoning::ReasoningError;
use vllm_parser::tool::{Tool, ToolCallDelta};
use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParserError, UnifiedParserOutput};
use vllm_text::DecodedText;
use vllm_tokenizer::test_utils::TestTokenizer;
use vllm_tokenizer::{TokenAnchor, TokenAttribution};
use super::{DecodedText, unified_event_stream};
use crate::event::AssistantBlockKind;
use super::unified_event_stream;
use crate::event::{AssistantBlockKind, ChatTokenUsage};
use crate::output::AssistantEvent;
enum ScriptedStep {
@@ -303,6 +338,7 @@ mod tests {
steps: VecDeque<ScriptedStep>,
reset_text: String,
tool_call_id: Option<String>,
finish_output: Option<UnifiedParserOutput>,
finish_error_reset_text: Option<String>,
}
@@ -312,10 +348,16 @@ mod tests {
steps: steps.into_iter().collect(),
reset_text: String::new(),
tool_call_id: Some("call_test".to_string()),
finish_output: None,
finish_error_reset_text: None,
}
}
fn with_finish_output(mut self, output: UnifiedParserOutput) -> Self {
self.finish_output = Some(output);
self
}
fn with_finish_error(mut self, reset_text: &str) -> Self {
self.finish_error_reset_text = Some(reset_text.to_string());
self
@@ -371,7 +413,7 @@ mod tests {
},
));
}
Ok(UnifiedParserOutput::default())
Ok(self.finish_output.take().unwrap_or_default())
}
fn reset(&mut self) -> String {
@@ -381,7 +423,7 @@ mod tests {
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
decoded: vllm_text::DecodedText::unattributed(delta),
decoded: DecodedText::unattributed(delta),
sampled: vllm_text::SampledDelta::default(),
finished: None,
}
@@ -389,7 +431,7 @@ mod tests {
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
vllm_text::output::DecodedTextEvent::TextDelta {
decoded: vllm_text::DecodedText::unattributed(delta),
decoded: DecodedText::unattributed(delta),
sampled: vllm_text::SampledDelta::default(),
finished: Some(Box::new(vllm_text::output::Finished {
usage: vllm_llm::TokenUsage::default(),
@@ -425,6 +467,22 @@ mod tests {
output
}
/// One reasoning piece carrying `tokens` synthetic token attributions.
fn attributed_reasoning(delta: &str, tokens: u32) -> UnifiedParserOutput {
let attributions = (0..tokens)
.map(|token_id| TokenAttribution {
token_id,
anchor: TokenAnchor::Visible { byte_offset: 0 },
})
.collect();
let mut output = UnifiedParserOutput::default();
output.push_reasoning(DecodedText {
text: delta.to_string(),
attributions,
});
output
}
fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput {
UnifiedParserOutput {
events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall(
@@ -455,6 +513,15 @@ mod tests {
output
}
/// Terminal usage with a default engine-level usage and the given final
/// reasoning token count.
fn done_usage(reasoning_tokens: usize) -> ChatTokenUsage {
ChatTokenUsage {
engine: vllm_llm::TokenUsage::default(),
reasoning_tokens,
}
}
#[tokio::test]
async fn unified_stream_parses_formatted_tool_call_without_latch() {
use vllm_parser::tool::{HermesToolParser, ToolParser as _};
@@ -519,6 +586,7 @@ mod tests {
vec![AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thinking".to_string(),
token_count: Some(0),
}]
);
}
@@ -565,6 +633,7 @@ mod tests {
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "thinking".to_string(),
token_count: Some(0),
},
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
@@ -594,6 +663,7 @@ mod tests {
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "visible ".to_string(),
token_count: None,
},
AssistantEvent::ToolCallStart {
id: "call_test".to_string(),
@@ -633,6 +703,7 @@ mod tests {
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: " done".to_string(),
token_count: None,
},
]
);
@@ -655,14 +726,17 @@ mod tests {
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "committed".to_string(),
token_count: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "buffered".to_string(),
token_count: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "later".to_string(),
token_count: None,
},
]
);
@@ -683,9 +757,134 @@ mod tests {
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "buffered".to_string(),
token_count: None,
},
AssistantEvent::Done {
usage: vllm_llm::TokenUsage::default(),
usage: done_usage(0),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn unified_stream_counts_reasoning_tokens_across_deltas() {
let events = collect(
ScriptedParser::new([
ScriptedStep::Output(attributed_reasoning("think", 2)),
ScriptedStep::Output(combined(text("visible"), attributed_reasoning("more", 3))),
]),
vec![decoded_delta("a"), finished_delta("b")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "think".to_string(),
token_count: Some(2),
},
// Non-reasoning deltas carry no token count.
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "visible".to_string(),
token_count: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "more".to_string(),
token_count: Some(3),
},
AssistantEvent::Done {
usage: done_usage(5),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn unified_stream_counts_reasoning_emitted_at_finish() {
let events = collect(
ScriptedParser::new([ScriptedStep::Output(text("visible"))])
.with_finish_output(attributed_reasoning("tail", 2)),
vec![finished_delta("raw")],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "visible".to_string(),
token_count: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "tail".to_string(),
token_count: Some(2),
},
AssistantEvent::Done {
usage: done_usage(2),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
},
]
);
}
#[tokio::test]
async fn unified_stream_fallback_keeps_emitted_reasoning_count() {
let events = collect(
ScriptedParser::new([
ScriptedStep::Output(attributed_reasoning("think", 2)),
ScriptedStep::Error {
committed: attributed_reasoning("more", 3),
reset_text: "buffered".to_string(),
},
]),
vec![
decoded_delta("a"),
decoded_delta("bad"),
finished_delta("later"),
],
)
.await;
assert_eq!(
events,
vec![
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "think".to_string(),
token_count: Some(2),
},
// Committed reasoning still counts before the failure fallback.
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "more".to_string(),
token_count: Some(3),
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "buffered".to_string(),
token_count: None,
},
AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "later".to_string(),
token_count: None,
},
AssistantEvent::Done {
usage: done_usage(5),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -727,9 +926,10 @@ mod tests {
kind: AssistantBlockKind::Text,
delta: "<|tool_call>call:write_file{content:<|\"|>hello world<|\"|>"
.to_string(),
token_count: None,
},
AssistantEvent::Done {
usage: vllm_llm::TokenUsage::default(),
usage: done_usage(0),
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
+7 -1
View File
@@ -259,6 +259,9 @@ impl HarmonyState {
events.push(AssistantEvent::TextDelta {
kind,
delta: "\n".to_string(),
// TODO: measure reasoning tokens from native
// analysis-channel token counts.
token_count: None,
});
}
@@ -268,6 +271,9 @@ impl HarmonyState {
events.push(AssistantEvent::TextDelta {
kind,
delta: group.text,
// TODO: measure reasoning tokens from native
// analysis-channel token counts.
token_count: None,
});
return;
}
@@ -373,7 +379,7 @@ async fn harmony_assistant_event_stream(
if let Some(finished) = finished {
y.yield_ok(AssistantEvent::Done {
usage: finished.usage,
usage: finished.usage.into(),
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
ec_transfer_params: finished.ec_transfer_params,
+5 -5
View File
@@ -13,7 +13,7 @@ use vllm_text::output::{
use super::*;
use crate::output::ChatOutputProcessor;
use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
use crate::{AssistantMessageExt, ChatEvent, FinishReason};
use crate::{AssistantMessageExt, ChatEvent, ChatTokenUsage, FinishReason};
fn assistant_prefix() -> Vec<u32> {
harmony_encoding()
@@ -125,11 +125,11 @@ fn interrupted_final_message_is_preserved() {
text: "hello".to_string(),
}],
},
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -176,11 +176,11 @@ fn interrupted_analysis_message_is_preserved() {
text: "think".to_string(),
}],
},
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
+4 -3
View File
@@ -7,12 +7,11 @@ use std::sync::Arc;
use futures::Stream;
use trait_set::trait_set;
use uuid::Uuid;
use vllm_llm::TokenUsage;
use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
use crate::FinishReason;
use crate::error::Result;
use crate::event::{AssistantBlockKind, ChatEvent};
use crate::event::{AssistantBlockKind, ChatEvent, ChatTokenUsage};
mod default;
mod harmony;
@@ -35,6 +34,8 @@ pub(crate) enum AssistantEvent {
TextDelta {
kind: AssistantBlockKind,
delta: String,
/// Number of generated tokens attributed to this delta, when available.
token_count: Option<usize>,
},
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
LogprobsDelta {
@@ -47,7 +48,7 @@ pub(crate) enum AssistantEvent {
/// `ToolCallStart`.
ToolCallArgumentsDelta { delta: String },
Done {
usage: TokenUsage,
usage: ChatTokenUsage,
finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
kv_transfer_params: Option<serde_json::Value>,
+81 -16
View File
@@ -16,6 +16,7 @@ use super::{AssistantEvent, AssistantEventStream};
use crate::error::Error;
use crate::event::{
AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantToolCall, ChatEvent,
ChatTokenUsage,
};
use crate::{FinishReason, Result};
@@ -80,10 +81,11 @@ impl StructuredEventState {
&mut self,
kind: AssistantBlockKind,
delta: String,
token_count: Option<usize>,
) -> Result<Vec<ChatEvent>> {
let mut events = Vec::new();
self.close_open_tool_call(&mut events);
self.push_text_delta(kind, delta, &mut events);
self.push_text_delta(kind, delta, token_count, &mut events);
Ok(events)
}
@@ -146,7 +148,7 @@ impl StructuredEventState {
/// Close any open block and emit the terminal `Done` event.
fn finish(
&mut self,
usage: vllm_llm::TokenUsage,
usage: ChatTokenUsage,
finish_reason: FinishReason,
kv_transfer_params: Option<serde_json::Value>,
ec_transfer_params: Option<serde_json::Value>,
@@ -170,6 +172,7 @@ impl StructuredEventState {
&mut self,
kind: AssistantBlockKind,
delta: String,
token_count: Option<usize>,
events: &mut Vec<ChatEvent>,
) {
if delta.is_empty() {
@@ -184,6 +187,7 @@ impl StructuredEventState {
index: open_block.index,
kind,
delta,
token_count,
});
}
// Otherwise, close the currently open block (if any) and start a
@@ -197,7 +201,12 @@ impl StructuredEventState {
text: delta.clone(),
});
events.push(ChatEvent::BlockStart { index, kind });
events.push(ChatEvent::BlockDelta { index, kind, delta });
events.push(ChatEvent::BlockDelta {
index,
kind,
delta,
token_count,
});
}
}
}
@@ -274,8 +283,12 @@ pub(crate) async fn structured_chat_event_stream(
})
.await;
}
AssistantEvent::TextDelta { kind, delta } => {
for next in state.process_text_delta(kind, delta)? {
AssistantEvent::TextDelta {
kind,
delta,
token_count,
} => {
for next in state.process_text_delta(kind, delta, token_count)? {
y.yield_ok(next).await;
}
}
@@ -321,7 +334,7 @@ mod tests {
use super::structured_chat_event_stream;
use crate::FinishReason;
use crate::error::Error;
use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent};
use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatTokenUsage};
use crate::output::AssistantEvent;
#[tokio::test]
@@ -335,11 +348,11 @@ mod tests {
delta: r#"{"city":"Paris"}"#.to_string(),
}),
Ok(AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -390,11 +403,11 @@ mod tests {
delta: r#"{"b":2}"#.to_string(),
}),
Ok(AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -433,6 +446,7 @@ mod tests {
Ok(AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "before".to_string(),
token_count: None,
}),
Ok(AssistantEvent::ToolCallStart {
id: "call_1".to_string(),
@@ -442,11 +456,11 @@ mod tests {
delta: r#"{"city":"Paris"}"#.to_string(),
}),
Ok(AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -492,13 +506,14 @@ mod tests {
Ok(AssistantEvent::TextDelta {
kind: AssistantBlockKind::Text,
delta: "done".to_string(),
token_count: None,
}),
Ok(AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -562,11 +577,11 @@ mod tests {
delta: r#"{"b":2}"#.to_string(),
}),
Ok(AssistantEvent::Done {
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -596,4 +611,54 @@ mod tests {
assert_eq!(tool_calls.len(), 1);
assert_eq!(tool_calls[0].name, "first");
}
#[tokio::test]
async fn structured_stream_forwards_reasoning_token_count() {
let events = stream::iter(vec![
Ok(AssistantEvent::Start {
prompt_token_ids: vec![].into(),
prompt_logprobs: None,
}),
Ok(AssistantEvent::TextDelta {
kind: AssistantBlockKind::Reasoning,
delta: "think".to_string(),
token_count: Some(3),
}),
Ok(AssistantEvent::Done {
usage: ChatTokenUsage {
engine: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 4,
cached_token_count: 0,
},
reasoning_tokens: 3,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
}),
]);
let events = structured_chat_event_stream(events, true)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<crate::Result<Vec<_>>>()
.unwrap();
assert!(matches!(events[0], ChatEvent::Start { .. }));
assert!(matches!(
events[2],
ChatEvent::BlockDelta {
kind: AssistantBlockKind::Reasoning,
token_count: Some(3),
..
}
));
let ChatEvent::Done { usage, .. } = &events[4] else {
panic!("expected done");
};
assert_eq!(usage.reasoning_tokens, 3);
assert_eq!(usage.output_token_count, 4);
}
}
+7 -7
View File
@@ -11,7 +11,7 @@ use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs}
use crate::FinishReason;
use crate::error::{Error, Result};
use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent};
use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent, ChatTokenUsage};
/// Final structured assistant message plus terminal stream metadata.
#[derive(Debug, Clone, PartialEq)]
@@ -21,7 +21,7 @@ pub struct CollectedAssistantMessage {
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
pub logprobs: Option<DecodedLogprobs>,
pub token_ids: Vec<u32>,
pub usage: vllm_llm::TokenUsage,
pub usage: ChatTokenUsage,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
@@ -140,7 +140,7 @@ mod tests {
use super::{ChatEventStream, CollectedAssistantMessage};
use crate::error::Error;
use crate::event::ChatEvent;
use crate::event::{ChatEvent, ChatTokenUsage};
#[tokio::test]
async fn collect_message_requires_terminal_done_event() {
@@ -195,11 +195,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
@@ -236,11 +236,11 @@ mod tests {
}],
}),
token_ids: vec![],
usage: vllm_llm::TokenUsage {
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
}),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
+15
View File
@@ -393,6 +393,7 @@ async fn chat_streams_text_events() {
index: 0,
kind: AssistantBlockKind::Text,
delta: "H".to_string(),
token_count: None,
}
);
assert_eq!(
@@ -401,6 +402,7 @@ async fn chat_streams_text_events() {
index: 0,
kind: AssistantBlockKind::Text,
delta: "i".to_string(),
token_count: None,
}
);
assert_eq!(
@@ -500,6 +502,7 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
index: 0,
kind: AssistantBlockKind::Text,
delta: "你".to_string(),
token_count: None,
}
);
assert_eq!(
@@ -586,6 +589,7 @@ async fn chat_stream_flushes_held_text_on_finish() {
index: 0,
kind: AssistantBlockKind::Text,
delta: "ok st".to_string(),
token_count: None,
}
);
assert_eq!(
@@ -773,6 +777,7 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
index: 0,
kind: AssistantBlockKind::Text,
delta: "Hi!".to_string(),
token_count: None,
}
);
assert_eq!(
@@ -880,6 +885,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
index: 0,
kind: AssistantBlockKind::Reasoning,
delta: "reason ".to_string(),
token_count: Some(7),
}
);
assert_eq!(
@@ -888,6 +894,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
index: 0,
kind: AssistantBlockKind::Reasoning,
delta: "more".to_string(),
token_count: Some(4),
}
);
assert_eq!(
@@ -912,6 +919,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
index: 1,
kind: AssistantBlockKind::Text,
delta: "answer".to_string(),
token_count: None,
}
);
assert_eq!(
@@ -927,12 +935,15 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
usage,
finish_reason,
..
})) => {
assert_eq!(message.reasoning().unwrap(), "reason more");
assert_eq!(message.text(), "answer");
assert_eq!(finish_reason, FinishReason::Length);
// 7 tokens for "reason " plus 4 for "more"; markers are excluded.
assert_eq!(usage.reasoning_tokens, 11);
}
other => panic!("unexpected final event: {other:?}"),
}
@@ -1000,6 +1011,8 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
message.usage.output_token_count,
"<think>inner</think>outer".len()
);
// 5 tokens for "inner"; markers are excluded.
assert_eq!(message.usage.reasoning_tokens, 5);
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
@@ -1079,6 +1092,7 @@ async fn chat_explicitly_disables_reasoning_parser() {
assert_eq!(message.message.reasoning(), None);
assert_eq!(message.message.text(), "<think>reason more</think>answer");
assert_eq!(message.finish_reason, FinishReason::Length);
assert_eq!(message.usage.reasoning_tokens, 0);
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
@@ -1379,6 +1393,7 @@ async fn chat_stream_and_collect_preserve_prompt_and_sample_logprobs() {
index: 0,
kind: AssistantBlockKind::Text,
delta: "H".to_string(),
token_count: None,
}
);
assert_eq!(