From d6bce42983bc0b2095ad6422dbf1399e219ae572 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 4 Sep 2026 09:47:24 +1000 Subject: [PATCH] [Rust Frontend] Report reasoning tokens in chat completion usage (#54883) Signed-off-by: Bugen Zhao --- .buildkite/test_areas/rust_frontend.yaml | 6 + rust/src/chat/src/event.rs | 35 ++- rust/src/chat/src/lib.rs | 2 +- rust/src/chat/src/output/default/unified.rs | 228 +++++++++++++- rust/src/chat/src/output/harmony/mod.rs | 8 +- rust/src/chat/src/output/harmony/tests.rs | 10 +- rust/src/chat/src/output/mod.rs | 7 +- rust/src/chat/src/output/structured.rs | 97 +++++- rust/src/chat/src/stream.rs | 14 +- rust/src/chat/tests/chat.rs | 15 + .../src/routes/openai/chat_completions.rs | 284 +++++++++++++++--- .../server/src/routes/openai/utils/types.rs | 76 ++++- .../server/src/routes/openai/utils/usage.rs | 20 +- .../chat_completion/test_include_reasoning.py | 18 +- .../chat_completion/test_serving_chat.py | 3 +- 15 files changed, 735 insertions(+), 88 deletions(-) diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 7867c1b22a..8ccc0c50e9 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -15,7 +15,9 @@ steps: - tests/utils.py - tests/benchmarks/test_serve_cli.py - tests/entrypoints/openai/chat_completion/test_chat_completion.py + - tests/entrypoints/openai/chat_completion/test_include_reasoning.py - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py + - tests/entrypoints/openai/chat_completion/test_serving_chat.py # - tests/entrypoints/openai/completion/test_prompt_validation.py - tests/entrypoints/launchers/test_shutdown.py @@ -27,7 +29,9 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming" + - pytest -v -s entrypoints/openai/chat_completion/test_include_reasoning.py - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" + - pytest -v -s entrypoints/openai/chat_completion/test_serving_chat.py -k "test_chat_per_request_metrics_follow_server_flag or test_streaming_reasoning_usage_counts_across_deltas or test_completion_tokens_details" # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds" - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" @@ -53,7 +57,9 @@ steps: - tests/utils.py - tests/benchmarks/test_serve_cli.py - tests/entrypoints/openai/chat_completion/test_chat_completion.py + - tests/entrypoints/openai/chat_completion/test_include_reasoning.py - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py + - tests/entrypoints/openai/chat_completion/test_serving_chat.py - tests/entrypoints/openai/completion/test_shutdown.py - tests/entrypoints/openai/test_return_token_ids.py - tests/entrypoints/openai/test_uds.py diff --git a/rust/src/chat/src/event.rs b/rust/src/chat/src/event.rs index 589fde7540..e57b8bbb86 100644 --- a/rust/src/chat/src/event.rs +++ b/rust/src/chat/src/event.rs @@ -123,6 +123,37 @@ impl [AssistantContentBlock] { } } +/// Chat-level token usage for one completed request. +/// +/// Extends the engine-level [`TokenUsage`] with reasoning attribution measured +/// by the chat output pipeline. [`TokenUsage`] itself stays engine-level and +/// is not touched by parsing. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ChatTokenUsage { + /// Engine-level token usage. + pub engine: TokenUsage, + /// Number of generated tokens attributed to reasoning; 0 when the + /// configured parser has no reasoning channel. + pub reasoning_tokens: usize, +} + +impl From for ChatTokenUsage { + fn from(engine: TokenUsage) -> Self { + Self { + engine, + reasoning_tokens: 0, + } + } +} + +impl Deref for ChatTokenUsage { + type Target = TokenUsage; + + fn deref(&self) -> &Self::Target { + &self.engine + } +} + /// Final structured assistant message assembled from the event stream. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct AssistantMessage { @@ -172,6 +203,8 @@ pub enum ChatEvent { index: usize, kind: AssistantBlockKind, delta: String, + /// Number of generated tokens attributed to this delta, when available. + token_count: Option, }, /// Per-decoded-update sample metadata: logprobs and/or output token IDs. LogprobsDelta { @@ -201,7 +234,7 @@ pub enum ChatEvent { /// metadata. Done { message: AssistantMessage, - usage: TokenUsage, + usage: ChatTokenUsage, finish_reason: FinishReason, /// Connector-specific KV transfer parameters for disaggregated serving. kv_transfer_params: Option, diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 8a7cac7b2c..85b98c4a19 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -18,7 +18,7 @@ pub use backend::{ pub use error::{Error, Result}; pub use event::{ AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantMessageExt, - AssistantToolCall, ChatEvent, + AssistantToolCall, ChatEvent, ChatTokenUsage, }; use futures::{StreamExt, TryStreamExt as _}; pub use llm_multimodal::MediaContentPart; diff --git a/rust/src/chat/src/output/default/unified.rs b/rust/src/chat/src/output/default/unified.rs index 33160fa992..964f0bf35a 100644 --- a/rust/src/chat/src/output/default/unified.rs +++ b/rust/src/chat/src/output/default/unified.rs @@ -17,7 +17,7 @@ use vllm_text::output::{DecodedText, DecodedTextEvent, SampledDelta}; use crate::Result; use crate::error::Error; -use crate::event::AssistantBlockKind; +use crate::event::{AssistantBlockKind, ChatTokenUsage}; use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id}; /// Per-stream unified parsing state. @@ -31,6 +31,10 @@ struct UnifiedParserState { /// Supported parsers currently emit at most one active tool call at a time. /// Change this to an indexed map if a model needs interleaved calls later. open_call_index: Option, + /// Running count of tokens attributed to reasoning. Kept solely to + /// produce the final count on `Done`; per-delta counts are carried by + /// each `TextDelta` event itself. + reasoning_tokens: usize, } impl UnifiedParserState { @@ -40,6 +44,7 @@ impl UnifiedParserState { parser, parser_failed: false, open_call_index: None, + reasoning_tokens: 0, } } @@ -69,10 +74,10 @@ impl UnifiedParserState { return Ok(text_event(AssistantBlockKind::Text, delta.text).into_iter().collect()); } - let mut output = UnifiedParserOutput::default(); // The parser consumes the delta; keep its text for the fallback path // that re-emits it as plain text. let fallback_text = delta.text.clone(); + let mut output = UnifiedParserOutput::default(); match self.parser.parse_into(delta, &mut output) { Ok(()) => { let mut events = Vec::new(); @@ -138,9 +143,13 @@ impl UnifiedParserState { self.open_call_index = None; push_text_delta(events, AssistantBlockKind::Text, delta); } - UnifiedParserEvent::Reasoning(delta) => { + UnifiedParserEvent::Reasoning(piece) => { self.open_call_index = None; - push_text_delta(events, AssistantBlockKind::Reasoning, delta.text); + // By construction this counts exactly the tokens emitted to + // the client as reasoning, so the parser-failure fallback + // needs no special handling. + self.reasoning_tokens += piece.attributions.len(); + push_piece_delta(events, AssistantBlockKind::Reasoning, piece); } UnifiedParserEvent::ToolCall(item) => { self.process_tool_item(item, events)?; @@ -200,12 +209,17 @@ impl UnifiedParserState { } } -/// Build one plain text event if `delta` is non-empty. +/// Build one plain text event with no measured tokens, if `delta` is +/// non-empty. fn text_event(kind: AssistantBlockKind, delta: String) -> Option { if delta.is_empty() { return None; } - Some(AssistantEvent::TextDelta { kind, delta }) + Some(AssistantEvent::TextDelta { + kind, + delta, + token_count: None, + }) } /// Push one plain text delta if it is non-empty. @@ -215,6 +229,22 @@ fn push_text_delta(events: &mut Vec, kind: AssistantBlockKind, d } } +/// Push one attributed delta, counting its tokens, if its text is non-empty. +fn push_piece_delta( + events: &mut Vec, + kind: AssistantBlockKind, + piece: DecodedText, +) { + if piece.text.is_empty() { + return; + } + events.push(AssistantEvent::TextDelta { + kind, + delta: piece.text, + token_count: Some(piece.attributions.len()), + }); +} + /// Wrap one decoded-text stream into the internal unified assistant stream. #[try_stream] pub(crate) async fn unified_event_stream( @@ -263,7 +293,10 @@ pub(crate) async fn unified_event_stream( y.yield_ok(next).await; } y.yield_ok(AssistantEvent::Done { - usage: finished.usage, + usage: ChatTokenUsage { + engine: finished.usage, + reasoning_tokens: state.reasoning_tokens, + }, finish_reason: finished.finish_reason, kv_transfer_params: finished.kv_transfer_params, ec_transfer_params: finished.ec_transfer_params, @@ -285,10 +318,12 @@ mod tests { use vllm_parser::reasoning::ReasoningError; use vllm_parser::tool::{Tool, ToolCallDelta}; use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParserError, UnifiedParserOutput}; + use vllm_text::DecodedText; use vllm_tokenizer::test_utils::TestTokenizer; + use vllm_tokenizer::{TokenAnchor, TokenAttribution}; - use super::{DecodedText, unified_event_stream}; - use crate::event::AssistantBlockKind; + use super::unified_event_stream; + use crate::event::{AssistantBlockKind, ChatTokenUsage}; use crate::output::AssistantEvent; enum ScriptedStep { @@ -303,6 +338,7 @@ mod tests { steps: VecDeque, reset_text: String, tool_call_id: Option, + finish_output: Option, finish_error_reset_text: Option, } @@ -312,10 +348,16 @@ mod tests { steps: steps.into_iter().collect(), reset_text: String::new(), tool_call_id: Some("call_test".to_string()), + finish_output: None, finish_error_reset_text: None, } } + fn with_finish_output(mut self, output: UnifiedParserOutput) -> Self { + self.finish_output = Some(output); + self + } + fn with_finish_error(mut self, reset_text: &str) -> Self { self.finish_error_reset_text = Some(reset_text.to_string()); self @@ -371,7 +413,7 @@ mod tests { }, )); } - Ok(UnifiedParserOutput::default()) + Ok(self.finish_output.take().unwrap_or_default()) } fn reset(&mut self) -> String { @@ -381,7 +423,7 @@ mod tests { fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent { vllm_text::output::DecodedTextEvent::TextDelta { - decoded: vllm_text::DecodedText::unattributed(delta), + decoded: DecodedText::unattributed(delta), sampled: vllm_text::SampledDelta::default(), finished: None, } @@ -389,7 +431,7 @@ mod tests { fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent { vllm_text::output::DecodedTextEvent::TextDelta { - decoded: vllm_text::DecodedText::unattributed(delta), + decoded: DecodedText::unattributed(delta), sampled: vllm_text::SampledDelta::default(), finished: Some(Box::new(vllm_text::output::Finished { usage: vllm_llm::TokenUsage::default(), @@ -425,6 +467,22 @@ mod tests { output } + /// One reasoning piece carrying `tokens` synthetic token attributions. + fn attributed_reasoning(delta: &str, tokens: u32) -> UnifiedParserOutput { + let attributions = (0..tokens) + .map(|token_id| TokenAttribution { + token_id, + anchor: TokenAnchor::Visible { byte_offset: 0 }, + }) + .collect(); + let mut output = UnifiedParserOutput::default(); + output.push_reasoning(DecodedText { + text: delta.to_string(), + attributions, + }); + output + } + fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput { UnifiedParserOutput { events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall( @@ -455,6 +513,15 @@ mod tests { output } + /// Terminal usage with a default engine-level usage and the given final + /// reasoning token count. + fn done_usage(reasoning_tokens: usize) -> ChatTokenUsage { + ChatTokenUsage { + engine: vllm_llm::TokenUsage::default(), + reasoning_tokens, + } + } + #[tokio::test] async fn unified_stream_parses_formatted_tool_call_without_latch() { use vllm_parser::tool::{HermesToolParser, ToolParser as _}; @@ -519,6 +586,7 @@ mod tests { vec![AssistantEvent::TextDelta { kind: AssistantBlockKind::Reasoning, delta: "thinking".to_string(), + token_count: Some(0), }] ); } @@ -565,6 +633,7 @@ mod tests { AssistantEvent::TextDelta { kind: AssistantBlockKind::Reasoning, delta: "thinking".to_string(), + token_count: Some(0), }, AssistantEvent::ToolCallStart { id: "call_test".to_string(), @@ -594,6 +663,7 @@ mod tests { AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "visible ".to_string(), + token_count: None, }, AssistantEvent::ToolCallStart { id: "call_test".to_string(), @@ -633,6 +703,7 @@ mod tests { AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: " done".to_string(), + token_count: None, }, ] ); @@ -655,14 +726,17 @@ mod tests { AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "committed".to_string(), + token_count: None, }, AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "buffered".to_string(), + token_count: None, }, AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "later".to_string(), + token_count: None, }, ] ); @@ -683,9 +757,134 @@ mod tests { AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "buffered".to_string(), + token_count: None, }, AssistantEvent::Done { - usage: vllm_llm::TokenUsage::default(), + usage: done_usage(0), + finish_reason: crate::FinishReason::Stop(None), + kv_transfer_params: None, + ec_transfer_params: None, + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_counts_reasoning_tokens_across_deltas() { + let events = collect( + ScriptedParser::new([ + ScriptedStep::Output(attributed_reasoning("think", 2)), + ScriptedStep::Output(combined(text("visible"), attributed_reasoning("more", 3))), + ]), + vec![decoded_delta("a"), finished_delta("b")], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "think".to_string(), + token_count: Some(2), + }, + // Non-reasoning deltas carry no token count. + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "visible".to_string(), + token_count: None, + }, + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "more".to_string(), + token_count: Some(3), + }, + AssistantEvent::Done { + usage: done_usage(5), + finish_reason: crate::FinishReason::Stop(None), + kv_transfer_params: None, + ec_transfer_params: None, + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_counts_reasoning_emitted_at_finish() { + let events = collect( + ScriptedParser::new([ScriptedStep::Output(text("visible"))]) + .with_finish_output(attributed_reasoning("tail", 2)), + vec![finished_delta("raw")], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "visible".to_string(), + token_count: None, + }, + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "tail".to_string(), + token_count: Some(2), + }, + AssistantEvent::Done { + usage: done_usage(2), + finish_reason: crate::FinishReason::Stop(None), + kv_transfer_params: None, + ec_transfer_params: None, + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_fallback_keeps_emitted_reasoning_count() { + let events = collect( + ScriptedParser::new([ + ScriptedStep::Output(attributed_reasoning("think", 2)), + ScriptedStep::Error { + committed: attributed_reasoning("more", 3), + reset_text: "buffered".to_string(), + }, + ]), + vec![ + decoded_delta("a"), + decoded_delta("bad"), + finished_delta("later"), + ], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "think".to_string(), + token_count: Some(2), + }, + // Committed reasoning still counts before the failure fallback. + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "more".to_string(), + token_count: Some(3), + }, + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "buffered".to_string(), + token_count: None, + }, + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "later".to_string(), + token_count: None, + }, + AssistantEvent::Done { + usage: done_usage(5), finish_reason: crate::FinishReason::Stop(None), kv_transfer_params: None, ec_transfer_params: None, @@ -727,9 +926,10 @@ mod tests { kind: AssistantBlockKind::Text, delta: "<|tool_call>call:write_file{content:<|\"|>hello world<|\"|>" .to_string(), + token_count: None, }, AssistantEvent::Done { - usage: vllm_llm::TokenUsage::default(), + usage: done_usage(0), finish_reason: crate::FinishReason::Stop(None), kv_transfer_params: None, ec_transfer_params: None, diff --git a/rust/src/chat/src/output/harmony/mod.rs b/rust/src/chat/src/output/harmony/mod.rs index 82f476efbe..01e0390622 100644 --- a/rust/src/chat/src/output/harmony/mod.rs +++ b/rust/src/chat/src/output/harmony/mod.rs @@ -259,6 +259,9 @@ impl HarmonyState { events.push(AssistantEvent::TextDelta { kind, delta: "\n".to_string(), + // TODO: measure reasoning tokens from native + // analysis-channel token counts. + token_count: None, }); } @@ -268,6 +271,9 @@ impl HarmonyState { events.push(AssistantEvent::TextDelta { kind, delta: group.text, + // TODO: measure reasoning tokens from native + // analysis-channel token counts. + token_count: None, }); return; } @@ -373,7 +379,7 @@ async fn harmony_assistant_event_stream( if let Some(finished) = finished { y.yield_ok(AssistantEvent::Done { - usage: finished.usage, + usage: finished.usage.into(), finish_reason: finished.finish_reason, kv_transfer_params: finished.kv_transfer_params, ec_transfer_params: finished.ec_transfer_params, diff --git a/rust/src/chat/src/output/harmony/tests.rs b/rust/src/chat/src/output/harmony/tests.rs index 4410bc8133..8dcf84dd98 100644 --- a/rust/src/chat/src/output/harmony/tests.rs +++ b/rust/src/chat/src/output/harmony/tests.rs @@ -13,7 +13,7 @@ use vllm_text::output::{ use super::*; use crate::output::ChatOutputProcessor; use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext}; -use crate::{AssistantMessageExt, ChatEvent, FinishReason}; +use crate::{AssistantMessageExt, ChatEvent, ChatTokenUsage, FinishReason}; fn assistant_prefix() -> Vec { harmony_encoding() @@ -125,11 +125,11 @@ fn interrupted_final_message_is_preserved() { text: "hello".to_string(), }], }, - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 0, output_token_count: 0, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -176,11 +176,11 @@ fn interrupted_analysis_message_is_preserved() { text: "think".to_string(), }], }, - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 0, output_token_count: 0, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, diff --git a/rust/src/chat/src/output/mod.rs b/rust/src/chat/src/output/mod.rs index ecf90015ca..dfccf3ba6f 100644 --- a/rust/src/chat/src/output/mod.rs +++ b/rust/src/chat/src/output/mod.rs @@ -7,12 +7,11 @@ use std::sync::Arc; use futures::Stream; use trait_set::trait_set; use uuid::Uuid; -use vllm_llm::TokenUsage; use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent}; use crate::FinishReason; use crate::error::Result; -use crate::event::{AssistantBlockKind, ChatEvent}; +use crate::event::{AssistantBlockKind, ChatEvent, ChatTokenUsage}; mod default; mod harmony; @@ -35,6 +34,8 @@ pub(crate) enum AssistantEvent { TextDelta { kind: AssistantBlockKind, delta: String, + /// Number of generated tokens attributed to this delta, when available. + token_count: Option, }, /// Per-decoded-update sample metadata: logprobs and/or output token IDs. LogprobsDelta { @@ -47,7 +48,7 @@ pub(crate) enum AssistantEvent { /// `ToolCallStart`. ToolCallArgumentsDelta { delta: String }, Done { - usage: TokenUsage, + usage: ChatTokenUsage, finish_reason: FinishReason, /// Connector-specific KV transfer parameters for disaggregated serving. kv_transfer_params: Option, diff --git a/rust/src/chat/src/output/structured.rs b/rust/src/chat/src/output/structured.rs index 387d8f9bd5..5f5bbf32fa 100644 --- a/rust/src/chat/src/output/structured.rs +++ b/rust/src/chat/src/output/structured.rs @@ -16,6 +16,7 @@ use super::{AssistantEvent, AssistantEventStream}; use crate::error::Error; use crate::event::{ AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantToolCall, ChatEvent, + ChatTokenUsage, }; use crate::{FinishReason, Result}; @@ -80,10 +81,11 @@ impl StructuredEventState { &mut self, kind: AssistantBlockKind, delta: String, + token_count: Option, ) -> Result> { let mut events = Vec::new(); self.close_open_tool_call(&mut events); - self.push_text_delta(kind, delta, &mut events); + self.push_text_delta(kind, delta, token_count, &mut events); Ok(events) } @@ -146,7 +148,7 @@ impl StructuredEventState { /// Close any open block and emit the terminal `Done` event. fn finish( &mut self, - usage: vllm_llm::TokenUsage, + usage: ChatTokenUsage, finish_reason: FinishReason, kv_transfer_params: Option, ec_transfer_params: Option, @@ -170,6 +172,7 @@ impl StructuredEventState { &mut self, kind: AssistantBlockKind, delta: String, + token_count: Option, events: &mut Vec, ) { if delta.is_empty() { @@ -184,6 +187,7 @@ impl StructuredEventState { index: open_block.index, kind, delta, + token_count, }); } // Otherwise, close the currently open block (if any) and start a @@ -197,7 +201,12 @@ impl StructuredEventState { text: delta.clone(), }); events.push(ChatEvent::BlockStart { index, kind }); - events.push(ChatEvent::BlockDelta { index, kind, delta }); + events.push(ChatEvent::BlockDelta { + index, + kind, + delta, + token_count, + }); } } } @@ -274,8 +283,12 @@ pub(crate) async fn structured_chat_event_stream( }) .await; } - AssistantEvent::TextDelta { kind, delta } => { - for next in state.process_text_delta(kind, delta)? { + AssistantEvent::TextDelta { + kind, + delta, + token_count, + } => { + for next in state.process_text_delta(kind, delta, token_count)? { y.yield_ok(next).await; } } @@ -321,7 +334,7 @@ mod tests { use super::structured_chat_event_stream; use crate::FinishReason; use crate::error::Error; - use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent}; + use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatTokenUsage}; use crate::output::AssistantEvent; #[tokio::test] @@ -335,11 +348,11 @@ mod tests { delta: r#"{"city":"Paris"}"#.to_string(), }), Ok(AssistantEvent::Done { - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -390,11 +403,11 @@ mod tests { delta: r#"{"b":2}"#.to_string(), }), Ok(AssistantEvent::Done { - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -433,6 +446,7 @@ mod tests { Ok(AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "before".to_string(), + token_count: None, }), Ok(AssistantEvent::ToolCallStart { id: "call_1".to_string(), @@ -442,11 +456,11 @@ mod tests { delta: r#"{"city":"Paris"}"#.to_string(), }), Ok(AssistantEvent::Done { - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -492,13 +506,14 @@ mod tests { Ok(AssistantEvent::TextDelta { kind: AssistantBlockKind::Text, delta: "done".to_string(), + token_count: None, }), Ok(AssistantEvent::Done { - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -562,11 +577,11 @@ mod tests { delta: r#"{"b":2}"#.to_string(), }), Ok(AssistantEvent::Done { - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 1, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -596,4 +611,54 @@ mod tests { assert_eq!(tool_calls.len(), 1); assert_eq!(tool_calls[0].name, "first"); } + + #[tokio::test] + async fn structured_stream_forwards_reasoning_token_count() { + let events = stream::iter(vec![ + Ok(AssistantEvent::Start { + prompt_token_ids: vec![].into(), + prompt_logprobs: None, + }), + Ok(AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "think".to_string(), + token_count: Some(3), + }), + Ok(AssistantEvent::Done { + usage: ChatTokenUsage { + engine: vllm_llm::TokenUsage { + prompt_token_count: 1, + output_token_count: 4, + cached_token_count: 0, + }, + reasoning_tokens: 3, + }, + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + ec_transfer_params: None, + }), + ]); + + let events = structured_chat_event_stream(events, true) + .collect::>() + .await + .into_iter() + .collect::>>() + .unwrap(); + + assert!(matches!(events[0], ChatEvent::Start { .. })); + assert!(matches!( + events[2], + ChatEvent::BlockDelta { + kind: AssistantBlockKind::Reasoning, + token_count: Some(3), + .. + } + )); + let ChatEvent::Done { usage, .. } = &events[4] else { + panic!("expected done"); + }; + assert_eq!(usage.reasoning_tokens, 3); + assert_eq!(usage.output_token_count, 4); + } } diff --git a/rust/src/chat/src/stream.rs b/rust/src/chat/src/stream.rs index c24dcaa84b..72b913078c 100644 --- a/rust/src/chat/src/stream.rs +++ b/rust/src/chat/src/stream.rs @@ -11,7 +11,7 @@ use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs} use crate::FinishReason; use crate::error::{Error, Result}; -use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent}; +use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent, ChatTokenUsage}; /// Final structured assistant message plus terminal stream metadata. #[derive(Debug, Clone, PartialEq)] @@ -21,7 +21,7 @@ pub struct CollectedAssistantMessage { pub prompt_logprobs: Option, pub logprobs: Option, pub token_ids: Vec, - pub usage: vllm_llm::TokenUsage, + pub usage: ChatTokenUsage, pub finish_reason: FinishReason, /// Connector-specific KV transfer parameters for disaggregated serving. pub kv_transfer_params: Option, @@ -140,7 +140,7 @@ mod tests { use super::{ChatEventStream, CollectedAssistantMessage}; use crate::error::Error; - use crate::event::ChatEvent; + use crate::event::{ChatEvent, ChatTokenUsage}; #[tokio::test] async fn collect_message_requires_terminal_done_event() { @@ -195,11 +195,11 @@ mod tests { }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -236,11 +236,11 @@ mod tests { }], }), token_ids: vec![], - usage: vllm_llm::TokenUsage { + usage: ChatTokenUsage::from(vllm_llm::TokenUsage { prompt_token_count: 2, output_token_count: 1, cached_token_count: 0, - }, + }), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index fd32506c6e..ab99b4d8ba 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -393,6 +393,7 @@ async fn chat_streams_text_events() { index: 0, kind: AssistantBlockKind::Text, delta: "H".to_string(), + token_count: None, } ); assert_eq!( @@ -401,6 +402,7 @@ async fn chat_streams_text_events() { index: 0, kind: AssistantBlockKind::Text, delta: "i".to_string(), + token_count: None, } ); assert_eq!( @@ -500,6 +502,7 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() { index: 0, kind: AssistantBlockKind::Text, delta: "你".to_string(), + token_count: None, } ); assert_eq!( @@ -586,6 +589,7 @@ async fn chat_stream_flushes_held_text_on_finish() { index: 0, kind: AssistantBlockKind::Text, delta: "ok st".to_string(), + token_count: None, } ); assert_eq!( @@ -773,6 +777,7 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() { index: 0, kind: AssistantBlockKind::Text, delta: "Hi!".to_string(), + token_count: None, } ); assert_eq!( @@ -880,6 +885,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() { index: 0, kind: AssistantBlockKind::Reasoning, delta: "reason ".to_string(), + token_count: Some(7), } ); assert_eq!( @@ -888,6 +894,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() { index: 0, kind: AssistantBlockKind::Reasoning, delta: "more".to_string(), + token_count: Some(4), } ); assert_eq!( @@ -912,6 +919,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() { index: 1, kind: AssistantBlockKind::Text, delta: "answer".to_string(), + token_count: None, } ); assert_eq!( @@ -927,12 +935,15 @@ async fn chat_stream_separates_reasoning_blocks_automatically() { match next_semantic(&mut stream).await { Some(Ok(ChatEvent::Done { message, + usage, finish_reason, .. })) => { assert_eq!(message.reasoning().unwrap(), "reason more"); assert_eq!(message.text(), "answer"); assert_eq!(finish_reason, FinishReason::Length); + // 7 tokens for "reason " plus 4 for "more"; markers are excluded. + assert_eq!(usage.reasoning_tokens, 11); } other => panic!("unexpected final event: {other:?}"), } @@ -1000,6 +1011,8 @@ async fn chat_collectors_return_structured_message_and_visible_text() { message.usage.output_token_count, "innerouter".len() ); + // 5 tokens for "inner"; markers are excluded. + assert_eq!(message.usage.reasoning_tokens, 5); let _ = shutdown_tx.send(()); engine_task.await.unwrap(); @@ -1079,6 +1092,7 @@ async fn chat_explicitly_disables_reasoning_parser() { assert_eq!(message.message.reasoning(), None); assert_eq!(message.message.text(), "reason moreanswer"); assert_eq!(message.finish_reason, FinishReason::Length); + assert_eq!(message.usage.reasoning_tokens, 0); let _ = shutdown_tx.send(()); engine_task.await.unwrap(); @@ -1379,6 +1393,7 @@ async fn chat_stream_and_collect_preserve_prompt_and_sample_logprobs() { index: 0, kind: AssistantBlockKind::Text, delta: "H".to_string(), + token_count: None, } ); assert_eq!( diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 40934e6095..474719a875 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -324,7 +324,20 @@ async fn chat_completion_chunk_stream( )); } } - Ok(ChatEvent::BlockDelta { kind, delta, .. }) => { + Ok(ChatEvent::BlockDelta { + kind, + delta, + token_count, + .. + }) => { + // The count is per-delta and follows the decoder clock; it is + // fed even when the reasoning delta itself is hidden from the + // client (`include_reasoning=false`). + if matches!(kind, AssistantBlockKind::Reasoning) + && let Some(token_count) = token_count + { + continuous_usage.add_reasoning_tokens(token_count); + } let include_delta = include_reasoning || !matches!(kind, AssistantBlockKind::Reasoning); if include_delta { @@ -433,6 +446,9 @@ async fn chat_completion_chunk_stream( final_usage.prompt_token_count, final_usage.output_token_count, ); + // Converge the running reasoning count onto the authoritative + // terminal value. + continuous_usage.set_reasoning_tokens(final_usage.reasoning_tokens); if let Some(pending_chunk) = pending_chunk.as_mut() && let Some(chunk) = pending_chunk.take_chunk(&envelope) @@ -806,16 +822,31 @@ mod tests { use futures::{StreamExt as _, stream}; use serde_json::json; use vllm_chat::{ - AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, FinishReason, + AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, ChatTokenUsage, + FinishReason, }; use vllm_engine_core_client::protocol::output::StopReason; use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob}; use super::{ - ApiServerOptions, ResponseOptions, StreamResponseEnvelope, block_delta_chunk, - chat_completion_chunk_stream, final_chunk, + ApiServerOptions, ChatCompletionStreamResponse, ResponseOptions, StreamResponseEnvelope, + block_delta_chunk, chat_completion_chunk_stream, final_chunk, }; + /// Terminal usage with the given engine-level counts and zero reasoning tokens. + fn done_usage( + prompt_tokens: usize, + output_tokens: usize, + cached_tokens: usize, + ) -> ChatTokenUsage { + vllm_llm::TokenUsage { + prompt_token_count: prompt_tokens, + output_token_count: output_tokens, + cached_token_count: cached_tokens, + } + .into() + } + fn stream_envelope() -> Arc { Arc::new(StreamResponseEnvelope::new( "chatcmpl-1".to_string(), @@ -913,6 +944,7 @@ mod tests { index: 0, kind: AssistantBlockKind::Text, delta: "hi".to_string(), + token_count: None, }), Ok(ChatEvent::LogprobsDelta { logprobs: Some(DecodedLogprobs { @@ -929,11 +961,7 @@ mod tests { }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 1, - }, + usage: done_usage(1, 1, 1), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -995,6 +1023,7 @@ mod tests { index: 0, kind: AssistantBlockKind::Reasoning, delta: "think".to_string(), + token_count: None, }), Ok(ChatEvent::LogprobsDelta { logprobs: Some(DecodedLogprobs { @@ -1011,11 +1040,7 @@ mod tests { }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 0, - }, + usage: done_usage(1, 1, 0), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -1059,19 +1084,17 @@ mod tests { index: 0, kind: AssistantBlockKind::Reasoning, delta: "think".to_string(), + token_count: None, }), Ok(ChatEvent::BlockDelta { index: 1, kind: AssistantBlockKind::Text, delta: "answer".to_string(), + token_count: None, }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 2, - cached_token_count: 0, - }, + usage: done_usage(1, 2, 0), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -1115,6 +1138,7 @@ mod tests { index: 0, kind: AssistantBlockKind::Reasoning, delta: "think".to_string(), + token_count: None, }), Ok(ChatEvent::LogprobsDelta { logprobs: Some(DecodedLogprobs { @@ -1133,6 +1157,7 @@ mod tests { index: 1, kind: AssistantBlockKind::Text, delta: "answer".to_string(), + token_count: None, }), Ok(ChatEvent::LogprobsDelta { logprobs: Some(DecodedLogprobs { @@ -1149,11 +1174,7 @@ mod tests { }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 2, - cached_token_count: 0, - }, + usage: done_usage(1, 2, 0), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -1226,6 +1247,7 @@ mod tests { index: 0, kind: AssistantBlockKind::Reasoning, delta: "think".to_string(), + token_count: None, }), Ok(ChatEvent::LogprobsDelta { logprobs: Some(DecodedLogprobs { @@ -1267,6 +1289,7 @@ mod tests { index: 1, kind: AssistantBlockKind::Text, delta: "answer".to_string(), + token_count: None, }), Ok(ChatEvent::LogprobsDelta { logprobs: Some(DecodedLogprobs { @@ -1283,11 +1306,7 @@ mod tests { }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 4, - cached_token_count: 0, - }, + usage: done_usage(1, 4, 0), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -1365,11 +1384,7 @@ mod tests { }), Ok(ChatEvent::Done { message: Default::default(), - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 0, - }, + usage: done_usage(1, 1, 0), finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, @@ -1410,4 +1425,205 @@ mod tests { None ); } + + /// Measured reasoning-token count of one chunk's continuous usage. + fn chunk_reasoning_tokens(chunk: &ChatCompletionStreamResponse) -> Option { + chunk + .usage + .as_ref() + .map(|usage| usage.completion_tokens_details.reasoning_tokens) + } + + #[tokio::test] + async fn chunk_stream_reports_reasoning_tokens_with_continuous_usage() { + let stream = stream::iter(vec![ + Ok(ChatEvent::Start { + prompt_token_ids: vec![7, 8, 9].into(), + prompt_logprobs: None, + }), + Ok(ChatEvent::BlockDelta { + index: 0, + kind: AssistantBlockKind::Reasoning, + delta: "think".to_string(), + token_count: Some(2), + }), + Ok(ChatEvent::LogprobsDelta { + logprobs: None, + token_ids: vec![20, 21], + }), + Ok(ChatEvent::BlockDelta { + index: 1, + kind: AssistantBlockKind::Text, + delta: "answer".to_string(), + // A stray text-side count must not pollute reasoning usage. + token_count: None, + }), + Ok(ChatEvent::Done { + message: Default::default(), + usage: ChatTokenUsage { + engine: vllm_llm::TokenUsage { + prompt_token_count: 3, + output_token_count: 3, + cached_token_count: 0, + }, + reasoning_tokens: 2, + }, + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + ec_transfer_params: None, + }), + ]); + + let chunks = chat_completion_chunk_stream( + stream, + "chatcmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + include_usage: true, + include_continuous_usage: true, + include_reasoning: true, + ..Default::default() + }, + ) + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("stream chunks"); + + // Role, reasoning delta, text delta, finish-reason chunk, usage chunk. + assert_eq!(chunks.len(), 5); + // The role chunk reports reasoning tokens from 0 onwards. + let role_usage = chunks[0].usage.as_ref().expect("role chunk usage"); + assert_eq!(role_usage.prompt_tokens, 3); + assert_eq!(role_usage.completion_tokens, Some(0)); + assert_eq!(chunk_reasoning_tokens(&chunks[0]), Some(0)); + // The reasoning delta contributes its per-delta measured count, fed by + // the `BlockDelta` itself rather than the sampled-clock `LogprobsDelta`. + assert_eq!(chunk_reasoning_tokens(&chunks[1]), Some(2)); + assert_eq!( + chunks[1].usage.as_ref().and_then(|usage| usage.completion_tokens), + Some(2) + ); + // The text delta's own count does not pollute reasoning usage, and the + // terminal usage chunk converges onto the authoritative final counts. + assert_eq!(chunk_reasoning_tokens(&chunks[2]), Some(2)); + let final_usage = chunks[4].usage.as_ref().expect("final usage chunk"); + assert_eq!(final_usage.prompt_tokens, 3); + assert_eq!(final_usage.completion_tokens, Some(3)); + assert_eq!(chunk_reasoning_tokens(&chunks[4]), Some(2)); + } + + #[tokio::test] + async fn chunk_stream_reports_reasoning_tokens_when_reasoning_hidden() { + let stream = stream::iter(vec![ + Ok(ChatEvent::Start { + prompt_token_ids: vec![].into(), + prompt_logprobs: None, + }), + Ok(ChatEvent::BlockDelta { + index: 0, + kind: AssistantBlockKind::Reasoning, + delta: "think".to_string(), + token_count: Some(2), + }), + Ok(ChatEvent::Done { + message: Default::default(), + usage: ChatTokenUsage { + engine: vllm_llm::TokenUsage { + prompt_token_count: 1, + output_token_count: 2, + cached_token_count: 0, + }, + reasoning_tokens: 2, + }, + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + ec_transfer_params: None, + }), + ]); + + let chunks = chat_completion_chunk_stream( + stream, + "chatcmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + include_usage: true, + include_continuous_usage: true, + ..Default::default() + }, + ) + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("stream chunks"); + + // The reasoning text stays hidden... + assert!( + chunks + .iter() + .all(|chunk| chunk.choices.iter().all(|choice| choice.delta.reasoning.is_none())) + ); + // ...but the count is still reported, from 0 on the role chunk up to + // the converged final usage. + assert_eq!(chunk_reasoning_tokens(&chunks[0]), Some(0)); + let final_usage = chunks.last().unwrap().usage.as_ref().expect("final usage chunk"); + assert_eq!(final_usage.completion_tokens, Some(2)); + assert_eq!(final_usage.completion_tokens_details.reasoning_tokens, 2); + } + + #[tokio::test] + async fn chunk_stream_reports_zero_reasoning_tokens_without_reasoning() { + let stream = stream::iter(vec![ + Ok(ChatEvent::Start { + prompt_token_ids: vec![].into(), + prompt_logprobs: None, + }), + Ok(ChatEvent::BlockDelta { + index: 0, + kind: AssistantBlockKind::Text, + delta: "answer".to_string(), + token_count: None, + }), + Ok(ChatEvent::Done { + message: Default::default(), + usage: done_usage(1, 1, 0), + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + ec_transfer_params: None, + }), + ]); + + let chunks = chat_completion_chunk_stream( + stream, + "chatcmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + include_usage: true, + include_continuous_usage: true, + ..Default::default() + }, + ) + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("stream chunks"); + + // Reasoning-token details are always reported; a stream without + // reasoning simply reports 0 on every usage payload. + for chunk in &chunks { + let usage = chunk.usage.as_ref().expect("usage"); + assert_eq!(usage.completion_tokens_details.reasoning_tokens, 0); + let json = serde_json::to_value(usage).expect("usage serializes"); + assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0); + } + } } diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 0ed56a6e6c..e9aab2d2b2 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -7,7 +7,7 @@ use std::slice; use llm_multimodal::ImageDetail; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; -use vllm_llm::TokenUsage; +use vllm_chat::ChatTokenUsage; // ============================================================================ // Constants @@ -440,14 +440,18 @@ pub struct Usage { pub total_tokens: usize, pub completion_tokens: Option, pub prompt_tokens_details: Option, + /// Reasoning-token breakdown. + /// Always present, 0 when the configured parser has no reasoning channel. + pub completion_tokens_details: CompletionTokenUsageInfo, } impl Usage { - /// Create a Usage with prompt-token cache details. + /// Create a Usage with prompt-token cache and reasoning-token details. pub fn from_counts( prompt_tokens: usize, completion_tokens: usize, cached_tokens: Option, + reasoning_tokens: usize, ) -> Self { Self { prompt_tokens, @@ -456,14 +460,20 @@ impl Usage { prompt_tokens_details: cached_tokens .filter(|&c| c > 0) .map(|c| PromptTokenUsageInfo { cached_tokens: c }), + completion_tokens_details: CompletionTokenUsageInfo { reasoning_tokens }, } } - pub fn from_token_usage(usage: TokenUsage, enable_prompt_tokens_details: bool) -> Self { + pub fn from_token_usage( + usage: impl Into, + enable_prompt_tokens_details: bool, + ) -> Self { + let usage = usage.into(); Self::from_counts( usage.prompt_token_count, usage.output_token_count, enable_prompt_tokens_details.then_some(usage.cached_token_count), + usage.reasoning_tokens, ) } } @@ -474,8 +484,15 @@ pub struct PromptTokenUsageInfo { pub cached_tokens: usize, } +/// Mirrors the Python vLLM `CompletionTokenUsageInfo` class. +#[derive(Debug, Clone, Default, Serialize)] +pub struct CompletionTokenUsageInfo { + pub reasoning_tokens: usize, +} + #[cfg(test)] mod usage_tests { + use vllm_chat::ChatTokenUsage; use vllm_llm::TokenUsage; use super::Usage; @@ -512,6 +529,59 @@ mod usage_tests { Some(3) ); } + + #[test] + fn token_usage_includes_reasoning_token_details() { + let usage = Usage::from_token_usage( + ChatTokenUsage { + engine: TokenUsage { + prompt_token_count: 5, + output_token_count: 4, + cached_token_count: 0, + }, + reasoning_tokens: 3, + }, + false, + ); + + assert_eq!(usage.completion_tokens_details.reasoning_tokens, 3); + let json = serde_json::to_value(&usage).expect("usage serializes"); + assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 3); + } + + #[test] + fn token_usage_serializes_zero_reasoning_tokens() { + let usage = Usage::from_token_usage( + ChatTokenUsage { + engine: TokenUsage { + prompt_token_count: 5, + output_token_count: 0, + cached_token_count: 0, + }, + reasoning_tokens: 0, + }, + false, + ); + + let json = serde_json::to_value(&usage).expect("usage serializes"); + assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0); + } + + #[test] + fn token_usage_reports_zero_reasoning_tokens_without_reasoning_parser() { + let usage = Usage::from_token_usage( + TokenUsage { + prompt_token_count: 5, + output_token_count: 2, + cached_token_count: 0, + }, + false, + ); + + assert_eq!(usage.completion_tokens_details.reasoning_tokens, 0); + let json = serde_json::to_value(&usage).expect("usage serializes"); + assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0); + } } /// OpenAI completions-style logprobs. diff --git a/rust/src/server/src/routes/openai/utils/usage.rs b/rust/src/server/src/routes/openai/utils/usage.rs index 845b7ddeb6..711b896629 100644 --- a/rust/src/server/src/routes/openai/utils/usage.rs +++ b/rust/src/server/src/routes/openai/utils/usage.rs @@ -12,6 +12,8 @@ use super::types::Usage; pub(crate) struct ContinuousUsage { prompt_tokens: usize, output_tokens: usize, + /// Running reasoning-token count, following the decoder clock. + reasoning_tokens: usize, } impl ContinuousUsage { @@ -25,6 +27,17 @@ impl ContinuousUsage { self.output_tokens = self.output_tokens.saturating_add(output_tokens); } + /// Add the tokens attributed to one reasoning delta to the running count. + pub(crate) fn add_reasoning_tokens(&mut self, token_count: usize) { + self.reasoning_tokens = self.reasoning_tokens.saturating_add(token_count); + } + + /// Converge the running reasoning count onto the authoritative terminal + /// value reported at stream end. + pub(crate) fn set_reasoning_tokens(&mut self, reasoning_tokens: usize) { + self.reasoning_tokens = reasoning_tokens; + } + /// Replace the running counts with the final counts reported by generation. pub(crate) fn set_final_counts(&mut self, prompt_tokens: usize, output_tokens: usize) { self.prompt_tokens = prompt_tokens; @@ -33,6 +46,11 @@ impl ContinuousUsage { /// Build a streaming usage snapshot without prompt cache details. pub(crate) fn to_usage(&self) -> Usage { - Usage::from_counts(self.prompt_tokens, self.output_tokens, None) + Usage::from_counts( + self.prompt_tokens, + self.output_tokens, + None, + self.reasoning_tokens, + ) } } diff --git a/tests/entrypoints/openai/chat_completion/test_include_reasoning.py b/tests/entrypoints/openai/chat_completion/test_include_reasoning.py index 50261cfff8..1d4e1ebc40 100644 --- a/tests/entrypoints/openai/chat_completion/test_include_reasoning.py +++ b/tests/entrypoints/openai/chat_completion/test_include_reasoning.py @@ -5,7 +5,7 @@ Verifies that reasoning content is included by default and suppressed when ``include_reasoning=False``, for both streaming and non-streaming -Chat Completions. +Chat Completions, while reasoning tokens remain represented in usage. """ import openai @@ -77,6 +77,22 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI) assert msg.content, "Expected content in response even without reasoning" +@pytest.mark.asyncio +async def test_reasoning_tokens_in_usage(client: openai.AsyncOpenAI): + """Hidden reasoning still contributes to terminal usage details.""" + response = await client.chat.completions.create( + model=MODEL_NAME, + messages=MESSAGES, + max_tokens=64, + extra_body={"include_reasoning": False}, + ) + + assert response.usage is not None + details = response.usage.completion_tokens_details + assert details is not None + assert details.reasoning_tokens > 0 + + @pytest.mark.asyncio async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI): """Default: reasoning deltas appear in streaming response.""" diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index f750830296..e7ea15877c 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -736,7 +736,8 @@ async def test_chat_per_request_metrics_follow_server_flag(): request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), ) assert disabled_response.metrics is None - assert disabled_response.usage.completion_tokens_details is None + details = disabled_response.usage.completion_tokens_details + assert details is None or details.reasoning_tokens == 0 enabled_serving = _build_minimal_metrics_serving_chat( enable_per_request_metrics=True