[Rust Frontend] Report reasoning tokens in chat completion usage (#54883)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -15,7 +15,9 @@ steps:
|
||||
- tests/utils.py
|
||||
- tests/benchmarks/test_serve_cli.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
||||
- tests/entrypoints/openai/chat_completion/test_include_reasoning.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
- tests/entrypoints/openai/chat_completion/test_serving_chat.py
|
||||
|
||||
# - tests/entrypoints/openai/completion/test_prompt_validation.py
|
||||
- tests/entrypoints/launchers/test_shutdown.py
|
||||
@@ -27,7 +29,9 @@ steps:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_include_reasoning.py
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_serving_chat.py -k "test_chat_per_request_metrics_follow_server_flag or test_streaming_reasoning_usage_counts_across_deltas or test_completion_tokens_details"
|
||||
|
||||
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
||||
- pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||
@@ -53,7 +57,9 @@ steps:
|
||||
- tests/utils.py
|
||||
- tests/benchmarks/test_serve_cli.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
||||
- tests/entrypoints/openai/chat_completion/test_include_reasoning.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
- tests/entrypoints/openai/chat_completion/test_serving_chat.py
|
||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
||||
- tests/entrypoints/openai/test_return_token_ids.py
|
||||
- tests/entrypoints/openai/test_uds.py
|
||||
|
||||
@@ -123,6 +123,37 @@ impl [AssistantContentBlock] {
|
||||
}
|
||||
}
|
||||
|
||||
/// Chat-level token usage for one completed request.
|
||||
///
|
||||
/// Extends the engine-level [`TokenUsage`] with reasoning attribution measured
|
||||
/// by the chat output pipeline. [`TokenUsage`] itself stays engine-level and
|
||||
/// is not touched by parsing.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct ChatTokenUsage {
|
||||
/// Engine-level token usage.
|
||||
pub engine: TokenUsage,
|
||||
/// Number of generated tokens attributed to reasoning; 0 when the
|
||||
/// configured parser has no reasoning channel.
|
||||
pub reasoning_tokens: usize,
|
||||
}
|
||||
|
||||
impl From<TokenUsage> for ChatTokenUsage {
|
||||
fn from(engine: TokenUsage) -> Self {
|
||||
Self {
|
||||
engine,
|
||||
reasoning_tokens: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for ChatTokenUsage {
|
||||
type Target = TokenUsage;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.engine
|
||||
}
|
||||
}
|
||||
|
||||
/// Final structured assistant message assembled from the event stream.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AssistantMessage {
|
||||
@@ -172,6 +203,8 @@ pub enum ChatEvent {
|
||||
index: usize,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
/// Number of generated tokens attributed to this delta, when available.
|
||||
token_count: Option<usize>,
|
||||
},
|
||||
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
|
||||
LogprobsDelta {
|
||||
@@ -201,7 +234,7 @@ pub enum ChatEvent {
|
||||
/// metadata.
|
||||
Done {
|
||||
message: AssistantMessage,
|
||||
usage: TokenUsage,
|
||||
usage: ChatTokenUsage,
|
||||
finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
|
||||
@@ -18,7 +18,7 @@ pub use backend::{
|
||||
pub use error::{Error, Result};
|
||||
pub use event::{
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantMessageExt,
|
||||
AssistantToolCall, ChatEvent,
|
||||
AssistantToolCall, ChatEvent, ChatTokenUsage,
|
||||
};
|
||||
use futures::{StreamExt, TryStreamExt as _};
|
||||
pub use llm_multimodal::MediaContentPart;
|
||||
|
||||
@@ -17,7 +17,7 @@ use vllm_text::output::{DecodedText, DecodedTextEvent, SampledDelta};
|
||||
|
||||
use crate::Result;
|
||||
use crate::error::Error;
|
||||
use crate::event::AssistantBlockKind;
|
||||
use crate::event::{AssistantBlockKind, ChatTokenUsage};
|
||||
use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id};
|
||||
|
||||
/// Per-stream unified parsing state.
|
||||
@@ -31,6 +31,10 @@ struct UnifiedParserState {
|
||||
/// Supported parsers currently emit at most one active tool call at a time.
|
||||
/// Change this to an indexed map if a model needs interleaved calls later.
|
||||
open_call_index: Option<usize>,
|
||||
/// Running count of tokens attributed to reasoning. Kept solely to
|
||||
/// produce the final count on `Done`; per-delta counts are carried by
|
||||
/// each `TextDelta` event itself.
|
||||
reasoning_tokens: usize,
|
||||
}
|
||||
|
||||
impl UnifiedParserState {
|
||||
@@ -40,6 +44,7 @@ impl UnifiedParserState {
|
||||
parser,
|
||||
parser_failed: false,
|
||||
open_call_index: None,
|
||||
reasoning_tokens: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,10 +74,10 @@ impl UnifiedParserState {
|
||||
return Ok(text_event(AssistantBlockKind::Text, delta.text).into_iter().collect());
|
||||
}
|
||||
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
// The parser consumes the delta; keep its text for the fallback path
|
||||
// that re-emits it as plain text.
|
||||
let fallback_text = delta.text.clone();
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
match self.parser.parse_into(delta, &mut output) {
|
||||
Ok(()) => {
|
||||
let mut events = Vec::new();
|
||||
@@ -138,9 +143,13 @@ impl UnifiedParserState {
|
||||
self.open_call_index = None;
|
||||
push_text_delta(events, AssistantBlockKind::Text, delta);
|
||||
}
|
||||
UnifiedParserEvent::Reasoning(delta) => {
|
||||
UnifiedParserEvent::Reasoning(piece) => {
|
||||
self.open_call_index = None;
|
||||
push_text_delta(events, AssistantBlockKind::Reasoning, delta.text);
|
||||
// By construction this counts exactly the tokens emitted to
|
||||
// the client as reasoning, so the parser-failure fallback
|
||||
// needs no special handling.
|
||||
self.reasoning_tokens += piece.attributions.len();
|
||||
push_piece_delta(events, AssistantBlockKind::Reasoning, piece);
|
||||
}
|
||||
UnifiedParserEvent::ToolCall(item) => {
|
||||
self.process_tool_item(item, events)?;
|
||||
@@ -200,12 +209,17 @@ impl UnifiedParserState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one plain text event if `delta` is non-empty.
|
||||
/// Build one plain text event with no measured tokens, if `delta` is
|
||||
/// non-empty.
|
||||
fn text_event(kind: AssistantBlockKind, delta: String) -> Option<AssistantEvent> {
|
||||
if delta.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(AssistantEvent::TextDelta { kind, delta })
|
||||
Some(AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta,
|
||||
token_count: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Push one plain text delta if it is non-empty.
|
||||
@@ -215,6 +229,22 @@ fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, d
|
||||
}
|
||||
}
|
||||
|
||||
/// Push one attributed delta, counting its tokens, if its text is non-empty.
|
||||
fn push_piece_delta(
|
||||
events: &mut Vec<AssistantEvent>,
|
||||
kind: AssistantBlockKind,
|
||||
piece: DecodedText,
|
||||
) {
|
||||
if piece.text.is_empty() {
|
||||
return;
|
||||
}
|
||||
events.push(AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta: piece.text,
|
||||
token_count: Some(piece.attributions.len()),
|
||||
});
|
||||
}
|
||||
|
||||
/// Wrap one decoded-text stream into the internal unified assistant stream.
|
||||
#[try_stream]
|
||||
pub(crate) async fn unified_event_stream(
|
||||
@@ -263,7 +293,10 @@ pub(crate) async fn unified_event_stream(
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
y.yield_ok(AssistantEvent::Done {
|
||||
usage: finished.usage,
|
||||
usage: ChatTokenUsage {
|
||||
engine: finished.usage,
|
||||
reasoning_tokens: state.reasoning_tokens,
|
||||
},
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
ec_transfer_params: finished.ec_transfer_params,
|
||||
@@ -285,10 +318,12 @@ mod tests {
|
||||
use vllm_parser::reasoning::ReasoningError;
|
||||
use vllm_parser::tool::{Tool, ToolCallDelta};
|
||||
use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
||||
use vllm_text::DecodedText;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use vllm_tokenizer::{TokenAnchor, TokenAttribution};
|
||||
|
||||
use super::{DecodedText, unified_event_stream};
|
||||
use crate::event::AssistantBlockKind;
|
||||
use super::unified_event_stream;
|
||||
use crate::event::{AssistantBlockKind, ChatTokenUsage};
|
||||
use crate::output::AssistantEvent;
|
||||
|
||||
enum ScriptedStep {
|
||||
@@ -303,6 +338,7 @@ mod tests {
|
||||
steps: VecDeque<ScriptedStep>,
|
||||
reset_text: String,
|
||||
tool_call_id: Option<String>,
|
||||
finish_output: Option<UnifiedParserOutput>,
|
||||
finish_error_reset_text: Option<String>,
|
||||
}
|
||||
|
||||
@@ -312,10 +348,16 @@ mod tests {
|
||||
steps: steps.into_iter().collect(),
|
||||
reset_text: String::new(),
|
||||
tool_call_id: Some("call_test".to_string()),
|
||||
finish_output: None,
|
||||
finish_error_reset_text: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_finish_output(mut self, output: UnifiedParserOutput) -> Self {
|
||||
self.finish_output = Some(output);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_finish_error(mut self, reset_text: &str) -> Self {
|
||||
self.finish_error_reset_text = Some(reset_text.to_string());
|
||||
self
|
||||
@@ -371,7 +413,7 @@ mod tests {
|
||||
},
|
||||
));
|
||||
}
|
||||
Ok(UnifiedParserOutput::default())
|
||||
Ok(self.finish_output.take().unwrap_or_default())
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
@@ -381,7 +423,7 @@ mod tests {
|
||||
|
||||
fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||
decoded: vllm_text::DecodedText::unattributed(delta),
|
||||
decoded: DecodedText::unattributed(delta),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: None,
|
||||
}
|
||||
@@ -389,7 +431,7 @@ mod tests {
|
||||
|
||||
fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
|
||||
vllm_text::output::DecodedTextEvent::TextDelta {
|
||||
decoded: vllm_text::DecodedText::unattributed(delta),
|
||||
decoded: DecodedText::unattributed(delta),
|
||||
sampled: vllm_text::SampledDelta::default(),
|
||||
finished: Some(Box::new(vllm_text::output::Finished {
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
@@ -425,6 +467,22 @@ mod tests {
|
||||
output
|
||||
}
|
||||
|
||||
/// One reasoning piece carrying `tokens` synthetic token attributions.
|
||||
fn attributed_reasoning(delta: &str, tokens: u32) -> UnifiedParserOutput {
|
||||
let attributions = (0..tokens)
|
||||
.map(|token_id| TokenAttribution {
|
||||
token_id,
|
||||
anchor: TokenAnchor::Visible { byte_offset: 0 },
|
||||
})
|
||||
.collect();
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
output.push_reasoning(DecodedText {
|
||||
text: delta.to_string(),
|
||||
attributions,
|
||||
});
|
||||
output
|
||||
}
|
||||
|
||||
fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput {
|
||||
UnifiedParserOutput {
|
||||
events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall(
|
||||
@@ -455,6 +513,15 @@ mod tests {
|
||||
output
|
||||
}
|
||||
|
||||
/// Terminal usage with a default engine-level usage and the given final
|
||||
/// reasoning token count.
|
||||
fn done_usage(reasoning_tokens: usize) -> ChatTokenUsage {
|
||||
ChatTokenUsage {
|
||||
engine: vllm_llm::TokenUsage::default(),
|
||||
reasoning_tokens,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unified_stream_parses_formatted_tool_call_without_latch() {
|
||||
use vllm_parser::tool::{HermesToolParser, ToolParser as _};
|
||||
@@ -519,6 +586,7 @@ mod tests {
|
||||
vec![AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "thinking".to_string(),
|
||||
token_count: Some(0),
|
||||
}]
|
||||
);
|
||||
}
|
||||
@@ -565,6 +633,7 @@ mod tests {
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "thinking".to_string(),
|
||||
token_count: Some(0),
|
||||
},
|
||||
AssistantEvent::ToolCallStart {
|
||||
id: "call_test".to_string(),
|
||||
@@ -594,6 +663,7 @@ mod tests {
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "visible ".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::ToolCallStart {
|
||||
id: "call_test".to_string(),
|
||||
@@ -633,6 +703,7 @@ mod tests {
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: " done".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
@@ -655,14 +726,17 @@ mod tests {
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "committed".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "buffered".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "later".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
@@ -683,9 +757,134 @@ mod tests {
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "buffered".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
usage: done_usage(0),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unified_stream_counts_reasoning_tokens_across_deltas() {
|
||||
let events = collect(
|
||||
ScriptedParser::new([
|
||||
ScriptedStep::Output(attributed_reasoning("think", 2)),
|
||||
ScriptedStep::Output(combined(text("visible"), attributed_reasoning("more", 3))),
|
||||
]),
|
||||
vec![decoded_delta("a"), finished_delta("b")],
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(2),
|
||||
},
|
||||
// Non-reasoning deltas carry no token count.
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "visible".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "more".to_string(),
|
||||
token_count: Some(3),
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
usage: done_usage(5),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unified_stream_counts_reasoning_emitted_at_finish() {
|
||||
let events = collect(
|
||||
ScriptedParser::new([ScriptedStep::Output(text("visible"))])
|
||||
.with_finish_output(attributed_reasoning("tail", 2)),
|
||||
vec![finished_delta("raw")],
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "visible".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "tail".to_string(),
|
||||
token_count: Some(2),
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
usage: done_usage(2),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unified_stream_fallback_keeps_emitted_reasoning_count() {
|
||||
let events = collect(
|
||||
ScriptedParser::new([
|
||||
ScriptedStep::Output(attributed_reasoning("think", 2)),
|
||||
ScriptedStep::Error {
|
||||
committed: attributed_reasoning("more", 3),
|
||||
reset_text: "buffered".to_string(),
|
||||
},
|
||||
]),
|
||||
vec![
|
||||
decoded_delta("a"),
|
||||
decoded_delta("bad"),
|
||||
finished_delta("later"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
events,
|
||||
vec![
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(2),
|
||||
},
|
||||
// Committed reasoning still counts before the failure fallback.
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "more".to_string(),
|
||||
token_count: Some(3),
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "buffered".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "later".to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
usage: done_usage(5),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -727,9 +926,10 @@ mod tests {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "<|tool_call>call:write_file{content:<|\"|>hello world<|\"|>"
|
||||
.to_string(),
|
||||
token_count: None,
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
usage: done_usage(0),
|
||||
finish_reason: crate::FinishReason::Stop(None),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
|
||||
@@ -259,6 +259,9 @@ impl HarmonyState {
|
||||
events.push(AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta: "\n".to_string(),
|
||||
// TODO: measure reasoning tokens from native
|
||||
// analysis-channel token counts.
|
||||
token_count: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -268,6 +271,9 @@ impl HarmonyState {
|
||||
events.push(AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta: group.text,
|
||||
// TODO: measure reasoning tokens from native
|
||||
// analysis-channel token counts.
|
||||
token_count: None,
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -373,7 +379,7 @@ async fn harmony_assistant_event_stream(
|
||||
|
||||
if let Some(finished) = finished {
|
||||
y.yield_ok(AssistantEvent::Done {
|
||||
usage: finished.usage,
|
||||
usage: finished.usage.into(),
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
ec_transfer_params: finished.ec_transfer_params,
|
||||
|
||||
@@ -13,7 +13,7 @@ use vllm_text::output::{
|
||||
use super::*;
|
||||
use crate::output::ChatOutputProcessor;
|
||||
use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
|
||||
use crate::{AssistantMessageExt, ChatEvent, FinishReason};
|
||||
use crate::{AssistantMessageExt, ChatEvent, ChatTokenUsage, FinishReason};
|
||||
|
||||
fn assistant_prefix() -> Vec<u32> {
|
||||
harmony_encoding()
|
||||
@@ -125,11 +125,11 @@ fn interrupted_final_message_is_preserved() {
|
||||
text: "hello".to_string(),
|
||||
}],
|
||||
},
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -176,11 +176,11 @@ fn interrupted_analysis_message_is_preserved() {
|
||||
text: "think".to_string(),
|
||||
}],
|
||||
},
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
|
||||
@@ -7,12 +7,11 @@ use std::sync::Arc;
|
||||
use futures::Stream;
|
||||
use trait_set::trait_set;
|
||||
use uuid::Uuid;
|
||||
use vllm_llm::TokenUsage;
|
||||
use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
|
||||
|
||||
use crate::FinishReason;
|
||||
use crate::error::Result;
|
||||
use crate::event::{AssistantBlockKind, ChatEvent};
|
||||
use crate::event::{AssistantBlockKind, ChatEvent, ChatTokenUsage};
|
||||
|
||||
mod default;
|
||||
mod harmony;
|
||||
@@ -35,6 +34,8 @@ pub(crate) enum AssistantEvent {
|
||||
TextDelta {
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
/// Number of generated tokens attributed to this delta, when available.
|
||||
token_count: Option<usize>,
|
||||
},
|
||||
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
|
||||
LogprobsDelta {
|
||||
@@ -47,7 +48,7 @@ pub(crate) enum AssistantEvent {
|
||||
/// `ToolCallStart`.
|
||||
ToolCallArgumentsDelta { delta: String },
|
||||
Done {
|
||||
usage: TokenUsage,
|
||||
usage: ChatTokenUsage,
|
||||
finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
|
||||
@@ -16,6 +16,7 @@ use super::{AssistantEvent, AssistantEventStream};
|
||||
use crate::error::Error;
|
||||
use crate::event::{
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantMessage, AssistantToolCall, ChatEvent,
|
||||
ChatTokenUsage,
|
||||
};
|
||||
use crate::{FinishReason, Result};
|
||||
|
||||
@@ -80,10 +81,11 @@ impl StructuredEventState {
|
||||
&mut self,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
token_count: Option<usize>,
|
||||
) -> Result<Vec<ChatEvent>> {
|
||||
let mut events = Vec::new();
|
||||
self.close_open_tool_call(&mut events);
|
||||
self.push_text_delta(kind, delta, &mut events);
|
||||
self.push_text_delta(kind, delta, token_count, &mut events);
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
@@ -146,7 +148,7 @@ impl StructuredEventState {
|
||||
/// Close any open block and emit the terminal `Done` event.
|
||||
fn finish(
|
||||
&mut self,
|
||||
usage: vllm_llm::TokenUsage,
|
||||
usage: ChatTokenUsage,
|
||||
finish_reason: FinishReason,
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
ec_transfer_params: Option<serde_json::Value>,
|
||||
@@ -170,6 +172,7 @@ impl StructuredEventState {
|
||||
&mut self,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
token_count: Option<usize>,
|
||||
events: &mut Vec<ChatEvent>,
|
||||
) {
|
||||
if delta.is_empty() {
|
||||
@@ -184,6 +187,7 @@ impl StructuredEventState {
|
||||
index: open_block.index,
|
||||
kind,
|
||||
delta,
|
||||
token_count,
|
||||
});
|
||||
}
|
||||
// Otherwise, close the currently open block (if any) and start a
|
||||
@@ -197,7 +201,12 @@ impl StructuredEventState {
|
||||
text: delta.clone(),
|
||||
});
|
||||
events.push(ChatEvent::BlockStart { index, kind });
|
||||
events.push(ChatEvent::BlockDelta { index, kind, delta });
|
||||
events.push(ChatEvent::BlockDelta {
|
||||
index,
|
||||
kind,
|
||||
delta,
|
||||
token_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -274,8 +283,12 @@ pub(crate) async fn structured_chat_event_stream(
|
||||
})
|
||||
.await;
|
||||
}
|
||||
AssistantEvent::TextDelta { kind, delta } => {
|
||||
for next in state.process_text_delta(kind, delta)? {
|
||||
AssistantEvent::TextDelta {
|
||||
kind,
|
||||
delta,
|
||||
token_count,
|
||||
} => {
|
||||
for next in state.process_text_delta(kind, delta, token_count)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
@@ -321,7 +334,7 @@ mod tests {
|
||||
use super::structured_chat_event_stream;
|
||||
use crate::FinishReason;
|
||||
use crate::error::Error;
|
||||
use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent};
|
||||
use crate::event::{AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatTokenUsage};
|
||||
use crate::output::AssistantEvent;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -335,11 +348,11 @@ mod tests {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -390,11 +403,11 @@ mod tests {
|
||||
delta: r#"{"b":2}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -433,6 +446,7 @@ mod tests {
|
||||
Ok(AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "before".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(AssistantEvent::ToolCallStart {
|
||||
id: "call_1".to_string(),
|
||||
@@ -442,11 +456,11 @@ mod tests {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -492,13 +506,14 @@ mod tests {
|
||||
Ok(AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "done".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -562,11 +577,11 @@ mod tests {
|
||||
delta: r#"{"b":2}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -596,4 +611,54 @@ mod tests {
|
||||
assert_eq!(tool_calls.len(), 1);
|
||||
assert_eq!(tool_calls[0].name, "first");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structured_stream_forwards_reasoning_token_count() {
|
||||
let events = stream::iter(vec![
|
||||
Ok(AssistantEvent::Start {
|
||||
prompt_token_ids: vec![].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(AssistantEvent::TextDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(3),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
usage: ChatTokenUsage {
|
||||
engine: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 4,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 3,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let events = structured_chat_event_stream(events, true)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<crate::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(events[0], ChatEvent::Start { .. }));
|
||||
assert!(matches!(
|
||||
events[2],
|
||||
ChatEvent::BlockDelta {
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
token_count: Some(3),
|
||||
..
|
||||
}
|
||||
));
|
||||
let ChatEvent::Done { usage, .. } = &events[4] else {
|
||||
panic!("expected done");
|
||||
};
|
||||
assert_eq!(usage.reasoning_tokens, 3);
|
||||
assert_eq!(usage.output_token_count, 4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs}
|
||||
|
||||
use crate::FinishReason;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent};
|
||||
use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent, ChatTokenUsage};
|
||||
|
||||
/// Final structured assistant message plus terminal stream metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -21,7 +21,7 @@ pub struct CollectedAssistantMessage {
|
||||
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub usage: vllm_llm::TokenUsage,
|
||||
pub usage: ChatTokenUsage,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
@@ -140,7 +140,7 @@ mod tests {
|
||||
|
||||
use super::{ChatEventStream, CollectedAssistantMessage};
|
||||
use crate::error::Error;
|
||||
use crate::event::ChatEvent;
|
||||
use crate::event::{ChatEvent, ChatTokenUsage};
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_message_requires_terminal_done_event() {
|
||||
@@ -195,11 +195,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -236,11 +236,11 @@ mod tests {
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
usage: vllm_llm::TokenUsage {
|
||||
usage: ChatTokenUsage::from(vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
}),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
|
||||
@@ -393,6 +393,7 @@ async fn chat_streams_text_events() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "H".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -401,6 +402,7 @@ async fn chat_streams_text_events() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "i".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -500,6 +502,7 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "你".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -586,6 +589,7 @@ async fn chat_stream_flushes_held_text_on_finish() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "ok st".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -773,6 +777,7 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "Hi!".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -880,6 +885,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "reason ".to_string(),
|
||||
token_count: Some(7),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -888,6 +894,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "more".to_string(),
|
||||
token_count: Some(4),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -912,6 +919,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -927,12 +935,15 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
usage,
|
||||
finish_reason,
|
||||
..
|
||||
})) => {
|
||||
assert_eq!(message.reasoning().unwrap(), "reason more");
|
||||
assert_eq!(message.text(), "answer");
|
||||
assert_eq!(finish_reason, FinishReason::Length);
|
||||
// 7 tokens for "reason " plus 4 for "more"; markers are excluded.
|
||||
assert_eq!(usage.reasoning_tokens, 11);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
}
|
||||
@@ -1000,6 +1011,8 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
|
||||
message.usage.output_token_count,
|
||||
"<think>inner</think>outer".len()
|
||||
);
|
||||
// 5 tokens for "inner"; markers are excluded.
|
||||
assert_eq!(message.usage.reasoning_tokens, 5);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
@@ -1079,6 +1092,7 @@ async fn chat_explicitly_disables_reasoning_parser() {
|
||||
assert_eq!(message.message.reasoning(), None);
|
||||
assert_eq!(message.message.text(), "<think>reason more</think>answer");
|
||||
assert_eq!(message.finish_reason, FinishReason::Length);
|
||||
assert_eq!(message.usage.reasoning_tokens, 0);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
@@ -1379,6 +1393,7 @@ async fn chat_stream_and_collect_preserve_prompt_and_sample_logprobs() {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "H".to_string(),
|
||||
token_count: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
|
||||
@@ -324,7 +324,20 @@ async fn chat_completion_chunk_stream(
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(ChatEvent::BlockDelta { kind, delta, .. }) => {
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
kind,
|
||||
delta,
|
||||
token_count,
|
||||
..
|
||||
}) => {
|
||||
// The count is per-delta and follows the decoder clock; it is
|
||||
// fed even when the reasoning delta itself is hidden from the
|
||||
// client (`include_reasoning=false`).
|
||||
if matches!(kind, AssistantBlockKind::Reasoning)
|
||||
&& let Some(token_count) = token_count
|
||||
{
|
||||
continuous_usage.add_reasoning_tokens(token_count);
|
||||
}
|
||||
let include_delta =
|
||||
include_reasoning || !matches!(kind, AssistantBlockKind::Reasoning);
|
||||
if include_delta {
|
||||
@@ -433,6 +446,9 @@ async fn chat_completion_chunk_stream(
|
||||
final_usage.prompt_token_count,
|
||||
final_usage.output_token_count,
|
||||
);
|
||||
// Converge the running reasoning count onto the authoritative
|
||||
// terminal value.
|
||||
continuous_usage.set_reasoning_tokens(final_usage.reasoning_tokens);
|
||||
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut()
|
||||
&& let Some(chunk) = pending_chunk.take_chunk(&envelope)
|
||||
@@ -806,16 +822,31 @@ mod tests {
|
||||
use futures::{StreamExt as _, stream};
|
||||
use serde_json::json;
|
||||
use vllm_chat::{
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, FinishReason,
|
||||
AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, ChatTokenUsage,
|
||||
FinishReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::{
|
||||
ApiServerOptions, ResponseOptions, StreamResponseEnvelope, block_delta_chunk,
|
||||
chat_completion_chunk_stream, final_chunk,
|
||||
ApiServerOptions, ChatCompletionStreamResponse, ResponseOptions, StreamResponseEnvelope,
|
||||
block_delta_chunk, chat_completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
/// Terminal usage with the given engine-level counts and zero reasoning tokens.
|
||||
fn done_usage(
|
||||
prompt_tokens: usize,
|
||||
output_tokens: usize,
|
||||
cached_tokens: usize,
|
||||
) -> ChatTokenUsage {
|
||||
vllm_llm::TokenUsage {
|
||||
prompt_token_count: prompt_tokens,
|
||||
output_token_count: output_tokens,
|
||||
cached_token_count: cached_tokens,
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
|
||||
Arc::new(StreamResponseEnvelope::new(
|
||||
"chatcmpl-1".to_string(),
|
||||
@@ -913,6 +944,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "hi".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -929,11 +961,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 1,
|
||||
},
|
||||
usage: done_usage(1, 1, 1),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -995,6 +1023,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1011,11 +1040,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 1, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1059,19 +1084,17 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 2, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1115,6 +1138,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1133,6 +1157,7 @@ mod tests {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1149,11 +1174,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 2, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1226,6 +1247,7 @@ mod tests {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1267,6 +1289,7 @@ mod tests {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: Some(DecodedLogprobs {
|
||||
@@ -1283,11 +1306,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 4,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 4, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1365,11 +1384,7 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
usage: done_usage(1, 1, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
@@ -1410,4 +1425,205 @@ mod tests {
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
/// Measured reasoning-token count of one chunk's continuous usage.
|
||||
fn chunk_reasoning_tokens(chunk: &ChatCompletionStreamResponse) -> Option<usize> {
|
||||
chunk
|
||||
.usage
|
||||
.as_ref()
|
||||
.map(|usage| usage.completion_tokens_details.reasoning_tokens)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunk_stream_reports_reasoning_tokens_with_continuous_usage() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![7, 8, 9].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(2),
|
||||
}),
|
||||
Ok(ChatEvent::LogprobsDelta {
|
||||
logprobs: None,
|
||||
token_ids: vec![20, 21],
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 1,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
// A stray text-side count must not pollute reasoning usage.
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: ChatTokenUsage {
|
||||
engine: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 3,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 2,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = chat_completion_chunk_stream(
|
||||
stream,
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
// Role, reasoning delta, text delta, finish-reason chunk, usage chunk.
|
||||
assert_eq!(chunks.len(), 5);
|
||||
// The role chunk reports reasoning tokens from 0 onwards.
|
||||
let role_usage = chunks[0].usage.as_ref().expect("role chunk usage");
|
||||
assert_eq!(role_usage.prompt_tokens, 3);
|
||||
assert_eq!(role_usage.completion_tokens, Some(0));
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[0]), Some(0));
|
||||
// The reasoning delta contributes its per-delta measured count, fed by
|
||||
// the `BlockDelta` itself rather than the sampled-clock `LogprobsDelta`.
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[1]), Some(2));
|
||||
assert_eq!(
|
||||
chunks[1].usage.as_ref().and_then(|usage| usage.completion_tokens),
|
||||
Some(2)
|
||||
);
|
||||
// The text delta's own count does not pollute reasoning usage, and the
|
||||
// terminal usage chunk converges onto the authoritative final counts.
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[2]), Some(2));
|
||||
let final_usage = chunks[4].usage.as_ref().expect("final usage chunk");
|
||||
assert_eq!(final_usage.prompt_tokens, 3);
|
||||
assert_eq!(final_usage.completion_tokens, Some(3));
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[4]), Some(2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunk_stream_reports_reasoning_tokens_when_reasoning_hidden() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Reasoning,
|
||||
delta: "think".to_string(),
|
||||
token_count: Some(2),
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: ChatTokenUsage {
|
||||
engine: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 2,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = chat_completion_chunk_stream(
|
||||
stream,
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
// The reasoning text stays hidden...
|
||||
assert!(
|
||||
chunks
|
||||
.iter()
|
||||
.all(|chunk| chunk.choices.iter().all(|choice| choice.delta.reasoning.is_none()))
|
||||
);
|
||||
// ...but the count is still reported, from 0 on the role chunk up to
|
||||
// the converged final usage.
|
||||
assert_eq!(chunk_reasoning_tokens(&chunks[0]), Some(0));
|
||||
let final_usage = chunks.last().unwrap().usage.as_ref().expect("final usage chunk");
|
||||
assert_eq!(final_usage.completion_tokens, Some(2));
|
||||
assert_eq!(final_usage.completion_tokens_details.reasoning_tokens, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chunk_stream_reports_zero_reasoning_tokens_without_reasoning() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(ChatEvent::Start {
|
||||
prompt_token_ids: vec![].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(ChatEvent::BlockDelta {
|
||||
index: 0,
|
||||
kind: AssistantBlockKind::Text,
|
||||
delta: "answer".to_string(),
|
||||
token_count: None,
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
usage: done_usage(1, 1, 0),
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = chat_completion_chunk_stream(
|
||||
stream,
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
// Reasoning-token details are always reported; a stream without
|
||||
// reasoning simply reports 0 on every usage payload.
|
||||
for chunk in &chunks {
|
||||
let usage = chunk.usage.as_ref().expect("usage");
|
||||
assert_eq!(usage.completion_tokens_details.reasoning_tokens, 0);
|
||||
let json = serde_json::to_value(usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::slice;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_llm::TokenUsage;
|
||||
use vllm_chat::ChatTokenUsage;
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
@@ -440,14 +440,18 @@ pub struct Usage {
|
||||
pub total_tokens: usize,
|
||||
pub completion_tokens: Option<usize>,
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
/// Reasoning-token breakdown.
|
||||
/// Always present, 0 when the configured parser has no reasoning channel.
|
||||
pub completion_tokens_details: CompletionTokenUsageInfo,
|
||||
}
|
||||
|
||||
impl Usage {
|
||||
/// Create a Usage with prompt-token cache details.
|
||||
/// Create a Usage with prompt-token cache and reasoning-token details.
|
||||
pub fn from_counts(
|
||||
prompt_tokens: usize,
|
||||
completion_tokens: usize,
|
||||
cached_tokens: Option<usize>,
|
||||
reasoning_tokens: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
prompt_tokens,
|
||||
@@ -456,14 +460,20 @@ impl Usage {
|
||||
prompt_tokens_details: cached_tokens
|
||||
.filter(|&c| c > 0)
|
||||
.map(|c| PromptTokenUsageInfo { cached_tokens: c }),
|
||||
completion_tokens_details: CompletionTokenUsageInfo { reasoning_tokens },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_token_usage(usage: TokenUsage, enable_prompt_tokens_details: bool) -> Self {
|
||||
pub fn from_token_usage(
|
||||
usage: impl Into<ChatTokenUsage>,
|
||||
enable_prompt_tokens_details: bool,
|
||||
) -> Self {
|
||||
let usage = usage.into();
|
||||
Self::from_counts(
|
||||
usage.prompt_token_count,
|
||||
usage.output_token_count,
|
||||
enable_prompt_tokens_details.then_some(usage.cached_token_count),
|
||||
usage.reasoning_tokens,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -474,8 +484,15 @@ pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: usize,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `CompletionTokenUsageInfo` class.
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct CompletionTokenUsageInfo {
|
||||
pub reasoning_tokens: usize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod usage_tests {
|
||||
use vllm_chat::ChatTokenUsage;
|
||||
use vllm_llm::TokenUsage;
|
||||
|
||||
use super::Usage;
|
||||
@@ -512,6 +529,59 @@ mod usage_tests {
|
||||
Some(3)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_includes_reasoning_token_details() {
|
||||
let usage = Usage::from_token_usage(
|
||||
ChatTokenUsage {
|
||||
engine: TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 4,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 3,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(usage.completion_tokens_details.reasoning_tokens, 3);
|
||||
let json = serde_json::to_value(&usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_serializes_zero_reasoning_tokens() {
|
||||
let usage = Usage::from_token_usage(
|
||||
ChatTokenUsage {
|
||||
engine: TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
reasoning_tokens: 0,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
let json = serde_json::to_value(&usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_reports_zero_reasoning_tokens_without_reasoning_parser() {
|
||||
let usage = Usage::from_token_usage(
|
||||
TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(usage.completion_tokens_details.reasoning_tokens, 0);
|
||||
let json = serde_json::to_value(&usage).expect("usage serializes");
|
||||
assert_eq!(json["completion_tokens_details"]["reasoning_tokens"], 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI completions-style logprobs.
|
||||
|
||||
@@ -12,6 +12,8 @@ use super::types::Usage;
|
||||
pub(crate) struct ContinuousUsage {
|
||||
prompt_tokens: usize,
|
||||
output_tokens: usize,
|
||||
/// Running reasoning-token count, following the decoder clock.
|
||||
reasoning_tokens: usize,
|
||||
}
|
||||
|
||||
impl ContinuousUsage {
|
||||
@@ -25,6 +27,17 @@ impl ContinuousUsage {
|
||||
self.output_tokens = self.output_tokens.saturating_add(output_tokens);
|
||||
}
|
||||
|
||||
/// Add the tokens attributed to one reasoning delta to the running count.
|
||||
pub(crate) fn add_reasoning_tokens(&mut self, token_count: usize) {
|
||||
self.reasoning_tokens = self.reasoning_tokens.saturating_add(token_count);
|
||||
}
|
||||
|
||||
/// Converge the running reasoning count onto the authoritative terminal
|
||||
/// value reported at stream end.
|
||||
pub(crate) fn set_reasoning_tokens(&mut self, reasoning_tokens: usize) {
|
||||
self.reasoning_tokens = reasoning_tokens;
|
||||
}
|
||||
|
||||
/// Replace the running counts with the final counts reported by generation.
|
||||
pub(crate) fn set_final_counts(&mut self, prompt_tokens: usize, output_tokens: usize) {
|
||||
self.prompt_tokens = prompt_tokens;
|
||||
@@ -33,6 +46,11 @@ impl ContinuousUsage {
|
||||
|
||||
/// Build a streaming usage snapshot without prompt cache details.
|
||||
pub(crate) fn to_usage(&self) -> Usage {
|
||||
Usage::from_counts(self.prompt_tokens, self.output_tokens, None)
|
||||
Usage::from_counts(
|
||||
self.prompt_tokens,
|
||||
self.output_tokens,
|
||||
None,
|
||||
self.reasoning_tokens,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
Verifies that reasoning content is included by default and suppressed
|
||||
when ``include_reasoning=False``, for both streaming and non-streaming
|
||||
Chat Completions.
|
||||
Chat Completions, while reasoning tokens remain represented in usage.
|
||||
"""
|
||||
|
||||
import openai
|
||||
@@ -77,6 +77,22 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI)
|
||||
assert msg.content, "Expected content in response even without reasoning"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_in_usage(client: openai.AsyncOpenAI):
|
||||
"""Hidden reasoning still contributes to terminal usage details."""
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=64,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
assert response.usage is not None
|
||||
details = response.usage.completion_tokens_details
|
||||
assert details is not None
|
||||
assert details.reasoning_tokens > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
|
||||
"""Default: reasoning deltas appear in streaming response."""
|
||||
|
||||
@@ -736,7 +736,8 @@ async def test_chat_per_request_metrics_follow_server_flag():
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||
)
|
||||
assert disabled_response.metrics is None
|
||||
assert disabled_response.usage.completion_tokens_details is None
|
||||
details = disabled_response.usage.completion_tokens_details
|
||||
assert details is None or details.reasoning_tokens == 0
|
||||
|
||||
enabled_serving = _build_minimal_metrics_serving_chat(
|
||||
enable_per_request_metrics=True
|
||||
|
||||
Reference in New Issue
Block a user