[Rust Frontend] Populate cached_token_count in responses (#44887)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-10 20:50:05 -07:00
committed by GitHub
parent 85a0ffae42
commit 5d5591d99b
34 changed files with 556 additions and 262 deletions
@@ -131,13 +131,13 @@ async fn main() -> Result<()> {
ChatEvent::LogprobsDelta { .. } => {}
ChatEvent::Done {
message,
output_token_count,
usage,
finish_reason: reason,
..
} => {
final_reasoning = message.reasoning().unwrap_or_default();
final_text = message.text();
final_output_token_count = output_token_count;
final_output_token_count = usage.output_token_count;
finish_reason = Some(reason);
break;
}
+2 -5
View File
@@ -2,6 +2,7 @@ use std::ops::Deref;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use vllm_llm::TokenUsage;
use vllm_text::{DecodedLogprobs, DecodedPromptLogprobs};
use crate::FinishReason;
@@ -197,11 +198,7 @@ pub enum ChatEvent {
/// metadata.
Done {
message: AssistantMessage,
/// Number of prompt tokens actually sent to the engine after chat
/// template rendering and tokenization.
prompt_token_count: usize,
/// Number of output tokens generated.
output_token_count: usize,
usage: TokenUsage,
finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
kv_transfer_params: Option<serde_json::Value>,
+11 -6
View File
@@ -178,8 +178,7 @@ pub(crate) async fn reasoning_event_stream(
y.yield_ok(next).await;
}
y.yield_ok(ContentEvent::Done {
prompt_token_count: finished.prompt_token_count,
output_token_count: finished.output_token_count,
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
})
@@ -289,8 +288,11 @@ mod tests {
token_ids: vec![],
logprobs: None,
finished: Some(vllm_text::Finished {
prompt_token_count: 3,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -322,8 +324,11 @@ mod tests {
delta: "def".to_string(),
},
ContentEvent::Done {
prompt_token_count: 3,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
+42 -20
View File
@@ -240,8 +240,7 @@ pub(crate) async fn tool_event_stream(
.await;
}
ContentEvent::Done {
prompt_token_count,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
} => {
@@ -250,8 +249,7 @@ pub(crate) async fn tool_event_stream(
}
y.yield_ok(AssistantEvent::Done {
prompt_token_count,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
})
@@ -465,8 +463,11 @@ mod tests {
})
})
.chain(std::iter::once(Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
})));
@@ -506,8 +507,11 @@ mod tests {
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -659,8 +663,11 @@ mod tests {
delta: "def".to_string(),
}),
Ok(ContentEvent::Done {
prompt_token_count: 3,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -697,8 +704,11 @@ mod tests {
delta: "def".to_string(),
},
AssistantEvent::Done {
prompt_token_count: 3,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 3,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
@@ -739,8 +749,11 @@ mod tests {
token_ids: vec![],
}),
Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -779,8 +792,11 @@ mod tests {
token_ids: vec![],
},
AssistantEvent::Done {
prompt_token_count: 1,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
},
@@ -796,8 +812,11 @@ mod tests {
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -901,8 +920,11 @@ mod tests {
delta: "ignored".to_string(),
}),
Ok(ContentEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
+1 -2
View File
@@ -366,8 +366,7 @@ async fn harmony_assistant_event_stream(
if let Some(finished) = finished {
y.yield_ok(AssistantEvent::Done {
prompt_token_count: finished.prompt_token_count,
output_token_count: finished.output_token_count,
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
})
+15 -6
View File
@@ -51,8 +51,11 @@ fn decoded_start() -> DecodedTextEvent {
fn finished() -> Finished {
Finished {
prompt_token_count: 0,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}
@@ -112,8 +115,11 @@ fn interrupted_final_message_is_preserved() {
text: "hello".to_string(),
}],
},
prompt_token_count: 0,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
})
@@ -171,8 +177,11 @@ fn interrupted_analysis_message_is_preserved() {
text: "think".to_string(),
}],
},
prompt_token_count: 0,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
})
+3 -4
View File
@@ -5,6 +5,7 @@ use futures::Stream;
use subenum::subenum;
use trait_set::trait_set;
use uuid::Uuid;
use vllm_llm::TokenUsage;
use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
use crate::FinishReason;
@@ -49,8 +50,7 @@ pub(crate) enum AssistantEvent {
ToolCallArgumentsDelta { delta: String },
#[subenum(ContentEvent)]
Done {
prompt_token_count: usize,
output_token_count: usize,
usage: TokenUsage,
finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
kv_transfer_params: Option<serde_json::Value>,
@@ -90,8 +90,7 @@ impl ContentEvent {
}
if let Some(finished) = finished {
events.push(Self::Done {
prompt_token_count: finished.prompt_token_count,
output_token_count: finished.output_token_count,
usage: finished.usage,
finish_reason: finished.finish_reason,
kv_transfer_params: finished.kv_transfer_params,
});
+24 -20
View File
@@ -127,8 +127,7 @@ impl StructuredEventState {
/// Close any open block and emit the terminal `Done` event.
fn finish(
&mut self,
prompt_token_count: usize,
output_token_count: usize,
usage: vllm_llm::TokenUsage,
finish_reason: FinishReason,
kv_transfer_params: Option<serde_json::Value>,
) -> Result<Vec<ChatEvent>> {
@@ -137,8 +136,7 @@ impl StructuredEventState {
self.close_open_tool_call(&mut events);
events.push(ChatEvent::Done {
message: self.message.clone(),
prompt_token_count,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
});
@@ -273,17 +271,11 @@ pub(crate) async fn structured_chat_event_stream(
}
}
AssistantEvent::Done {
prompt_token_count,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
} => {
for next in state.finish(
prompt_token_count,
output_token_count,
finish_reason,
kv_transfer_params,
)? {
for next in state.finish(usage, finish_reason, kv_transfer_params)? {
y.yield_ok(next).await;
}
}
@@ -313,8 +305,11 @@ mod tests {
delta: r#"{"city":"Paris"}"#.to_string(),
}),
Ok(AssistantEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -364,8 +359,11 @@ mod tests {
delta: r#"{"b":2}"#.to_string(),
}),
Ok(AssistantEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -412,8 +410,11 @@ mod tests {
delta: r#"{"city":"Paris"}"#.to_string(),
}),
Ok(AssistantEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -460,8 +461,11 @@ mod tests {
delta: "done".to_string(),
}),
Ok(AssistantEvent::Done {
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
+13 -10
View File
@@ -14,12 +14,11 @@ use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent};
#[derive(Debug, Clone, PartialEq)]
pub struct CollectedAssistantMessage {
pub message: AssistantMessage,
pub prompt_token_count: usize,
pub prompt_token_ids: Arc<[u32]>,
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
pub logprobs: Option<DecodedLogprobs>,
pub token_ids: Vec<u32>,
pub output_token_count: usize,
pub usage: vllm_llm::TokenUsage,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
@@ -75,21 +74,19 @@ impl ChatEventStream {
}
ChatEvent::Done {
message: done,
prompt_token_count,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
} => {
return Ok(CollectedAssistantMessage {
message: done,
prompt_token_count,
prompt_token_ids,
prompt_logprobs,
logprobs: (!logprob_positions.is_empty()).then_some(DecodedLogprobs {
positions: logprob_positions,
}),
token_ids,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
});
@@ -190,8 +187,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 2,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -203,7 +203,6 @@ mod tests {
collected,
CollectedAssistantMessage {
message: Default::default(),
prompt_token_count: 2,
prompt_token_ids: vec![10, 11].into(),
prompt_logprobs: Some(DecodedPromptLogprobs {
first_token_id: 0,
@@ -228,7 +227,11 @@ mod tests {
}],
}),
token_ids: vec![],
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}
+10 -18
View File
@@ -494,12 +494,12 @@ async fn chat_streams_text_events() {
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
usage,
finish_reason,
..
})) => {
assert_eq!(message.text(), "Hi");
assert_eq!(output_token_count, 3);
assert_eq!(usage.output_token_count, 3);
assert_eq!(
finish_reason,
FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))
@@ -590,13 +590,9 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
);
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
..
})) => {
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
assert_eq!(message.text(), "你");
assert_eq!(output_token_count, 4);
assert_eq!(usage.output_token_count, 4);
}
other => panic!("unexpected final event: {other:?}"),
}
@@ -681,12 +677,12 @@ async fn chat_stream_flushes_held_text_on_finish() {
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
usage,
finish_reason,
..
})) => {
assert_eq!(message.text(), "ok st");
assert_eq!(output_token_count, 5);
assert_eq!(usage.output_token_count, 5);
assert_eq!(finish_reason, FinishReason::Length);
}
other => panic!("unexpected final event: {other:?}"),
@@ -857,13 +853,9 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
);
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
..
})) => {
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
assert_eq!(message.text(), "Hi!");
assert_eq!(output_token_count, 3);
assert_eq!(usage.output_token_count, 3);
}
other => panic!("unexpected final event: {other:?}"),
}
@@ -1066,11 +1058,11 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
assert_eq!(message.message.text(), "outer");
assert_eq!(message.finish_reason, FinishReason::Length);
assert_eq!(
message.prompt_token_count,
message.usage.prompt_token_count,
"system: You are terse.\nuser: Say hi\nassistant:".len()
);
assert_eq!(
message.output_token_count,
message.usage.output_token_count,
"<think>inner</think>outer".len()
);
+10 -4
View File
@@ -508,8 +508,11 @@ fn decoded_completion_stream(
token_ids: Vec::new(),
logprobs: None,
finished: Some(Finished {
prompt_token_count: 0,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -519,8 +522,11 @@ fn decoded_completion_stream(
let last_index = chunks.len() - 1;
for (index, chunk) in chunks.into_iter().enumerate() {
let finished = (index == last_index).then(|| Finished {
prompt_token_count,
output_token_count: completion_body.chars().count(),
usage: vllm_llm::TokenUsage {
prompt_token_count,
output_token_count: completion_body.chars().count(),
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
});