[Rust Frontend] Populate cached_token_count in responses (#44887)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -131,13 +131,13 @@ async fn main() -> Result<()> {
|
||||
ChatEvent::LogprobsDelta { .. } => {}
|
||||
ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason: reason,
|
||||
..
|
||||
} => {
|
||||
final_reasoning = message.reasoning().unwrap_or_default();
|
||||
final_text = message.text();
|
||||
final_output_token_count = output_token_count;
|
||||
final_output_token_count = usage.output_token_count;
|
||||
finish_reason = Some(reason);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_llm::TokenUsage;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPromptLogprobs};
|
||||
|
||||
use crate::FinishReason;
|
||||
@@ -197,11 +198,7 @@ pub enum ChatEvent {
|
||||
/// metadata.
|
||||
Done {
|
||||
message: AssistantMessage,
|
||||
/// Number of prompt tokens actually sent to the engine after chat
|
||||
/// template rendering and tokenization.
|
||||
prompt_token_count: usize,
|
||||
/// Number of output tokens generated.
|
||||
output_token_count: usize,
|
||||
usage: TokenUsage,
|
||||
finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
|
||||
@@ -178,8 +178,7 @@ pub(crate) async fn reasoning_event_stream(
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
y.yield_ok(ContentEvent::Done {
|
||||
prompt_token_count: finished.prompt_token_count,
|
||||
output_token_count: finished.output_token_count,
|
||||
usage: finished.usage,
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
})
|
||||
@@ -289,8 +288,11 @@ mod tests {
|
||||
token_ids: vec![],
|
||||
logprobs: None,
|
||||
finished: Some(vllm_text::Finished {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -322,8 +324,11 @@ mod tests {
|
||||
delta: "def".to_string(),
|
||||
},
|
||||
ContentEvent::Done {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
},
|
||||
|
||||
@@ -240,8 +240,7 @@ pub(crate) async fn tool_event_stream(
|
||||
.await;
|
||||
}
|
||||
ContentEvent::Done {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} => {
|
||||
@@ -250,8 +249,7 @@ pub(crate) async fn tool_event_stream(
|
||||
}
|
||||
|
||||
y.yield_ok(AssistantEvent::Done {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
})
|
||||
@@ -465,8 +463,11 @@ mod tests {
|
||||
})
|
||||
})
|
||||
.chain(std::iter::once(Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
})));
|
||||
@@ -506,8 +507,11 @@ mod tests {
|
||||
delta: "ignored".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -659,8 +663,11 @@ mod tests {
|
||||
delta: "def".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -697,8 +704,11 @@ mod tests {
|
||||
delta: "def".to_string(),
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 3,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
},
|
||||
@@ -739,8 +749,11 @@ mod tests {
|
||||
token_ids: vec![],
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -779,8 +792,11 @@ mod tests {
|
||||
token_ids: vec![],
|
||||
},
|
||||
AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
},
|
||||
@@ -796,8 +812,11 @@ mod tests {
|
||||
delta: "ignored".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -901,8 +920,11 @@ mod tests {
|
||||
delta: "ignored".to_string(),
|
||||
}),
|
||||
Ok(ContentEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
|
||||
@@ -366,8 +366,7 @@ async fn harmony_assistant_event_stream(
|
||||
|
||||
if let Some(finished) = finished {
|
||||
y.yield_ok(AssistantEvent::Done {
|
||||
prompt_token_count: finished.prompt_token_count,
|
||||
output_token_count: finished.output_token_count,
|
||||
usage: finished.usage,
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
})
|
||||
|
||||
@@ -51,8 +51,11 @@ fn decoded_start() -> DecodedTextEvent {
|
||||
|
||||
fn finished() -> Finished {
|
||||
Finished {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
@@ -112,8 +115,11 @@ fn interrupted_final_message_is_preserved() {
|
||||
text: "hello".to_string(),
|
||||
}],
|
||||
},
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
})
|
||||
@@ -171,8 +177,11 @@ fn interrupted_analysis_message_is_preserved() {
|
||||
text: "think".to_string(),
|
||||
}],
|
||||
},
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
})
|
||||
|
||||
@@ -5,6 +5,7 @@ use futures::Stream;
|
||||
use subenum::subenum;
|
||||
use trait_set::trait_set;
|
||||
use uuid::Uuid;
|
||||
use vllm_llm::TokenUsage;
|
||||
use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
|
||||
|
||||
use crate::FinishReason;
|
||||
@@ -49,8 +50,7 @@ pub(crate) enum AssistantEvent {
|
||||
ToolCallArgumentsDelta { delta: String },
|
||||
#[subenum(ContentEvent)]
|
||||
Done {
|
||||
prompt_token_count: usize,
|
||||
output_token_count: usize,
|
||||
usage: TokenUsage,
|
||||
finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
@@ -90,8 +90,7 @@ impl ContentEvent {
|
||||
}
|
||||
if let Some(finished) = finished {
|
||||
events.push(Self::Done {
|
||||
prompt_token_count: finished.prompt_token_count,
|
||||
output_token_count: finished.output_token_count,
|
||||
usage: finished.usage,
|
||||
finish_reason: finished.finish_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params,
|
||||
});
|
||||
|
||||
@@ -127,8 +127,7 @@ impl StructuredEventState {
|
||||
/// Close any open block and emit the terminal `Done` event.
|
||||
fn finish(
|
||||
&mut self,
|
||||
prompt_token_count: usize,
|
||||
output_token_count: usize,
|
||||
usage: vllm_llm::TokenUsage,
|
||||
finish_reason: FinishReason,
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
) -> Result<Vec<ChatEvent>> {
|
||||
@@ -137,8 +136,7 @@ impl StructuredEventState {
|
||||
self.close_open_tool_call(&mut events);
|
||||
events.push(ChatEvent::Done {
|
||||
message: self.message.clone(),
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
});
|
||||
@@ -273,17 +271,11 @@ pub(crate) async fn structured_chat_event_stream(
|
||||
}
|
||||
}
|
||||
AssistantEvent::Done {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} => {
|
||||
for next in state.finish(
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
)? {
|
||||
for next in state.finish(usage, finish_reason, kv_transfer_params)? {
|
||||
y.yield_ok(next).await;
|
||||
}
|
||||
}
|
||||
@@ -313,8 +305,11 @@ mod tests {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -364,8 +359,11 @@ mod tests {
|
||||
delta: r#"{"b":2}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -412,8 +410,11 @@ mod tests {
|
||||
delta: r#"{"city":"Paris"}"#.to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -460,8 +461,11 @@ mod tests {
|
||||
delta: "done".to_string(),
|
||||
}),
|
||||
Ok(AssistantEvent::Done {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
|
||||
+13
-10
@@ -14,12 +14,11 @@ use crate::event::{AssistantContentBlock, AssistantMessage, ChatEvent};
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CollectedAssistantMessage {
|
||||
pub message: AssistantMessage,
|
||||
pub prompt_token_count: usize,
|
||||
pub prompt_token_ids: Arc<[u32]>,
|
||||
pub prompt_logprobs: Option<DecodedPromptLogprobs>,
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub output_token_count: usize,
|
||||
pub usage: vllm_llm::TokenUsage,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
@@ -75,21 +74,19 @@ impl ChatEventStream {
|
||||
}
|
||||
ChatEvent::Done {
|
||||
message: done,
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} => {
|
||||
return Ok(CollectedAssistantMessage {
|
||||
message: done,
|
||||
prompt_token_count,
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
logprobs: (!logprob_positions.is_empty()).then_some(DecodedLogprobs {
|
||||
positions: logprob_positions,
|
||||
}),
|
||||
token_ids,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
});
|
||||
@@ -190,8 +187,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -203,7 +203,6 @@ mod tests {
|
||||
collected,
|
||||
CollectedAssistantMessage {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 2,
|
||||
prompt_token_ids: vec![10, 11].into(),
|
||||
prompt_logprobs: Some(DecodedPromptLogprobs {
|
||||
first_token_id: 0,
|
||||
@@ -228,7 +227,11 @@ mod tests {
|
||||
}],
|
||||
}),
|
||||
token_ids: vec![],
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
|
||||
+10
-18
@@ -494,12 +494,12 @@ async fn chat_streams_text_events() {
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
..
|
||||
})) => {
|
||||
assert_eq!(message.text(), "Hi");
|
||||
assert_eq!(output_token_count, 3);
|
||||
assert_eq!(usage.output_token_count, 3);
|
||||
assert_eq!(
|
||||
finish_reason,
|
||||
FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))
|
||||
@@ -590,13 +590,9 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
|
||||
);
|
||||
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
..
|
||||
})) => {
|
||||
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
|
||||
assert_eq!(message.text(), "你");
|
||||
assert_eq!(output_token_count, 4);
|
||||
assert_eq!(usage.output_token_count, 4);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
}
|
||||
@@ -681,12 +677,12 @@ async fn chat_stream_flushes_held_text_on_finish() {
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
..
|
||||
})) => {
|
||||
assert_eq!(message.text(), "ok st");
|
||||
assert_eq!(output_token_count, 5);
|
||||
assert_eq!(usage.output_token_count, 5);
|
||||
assert_eq!(finish_reason, FinishReason::Length);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
@@ -857,13 +853,9 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
|
||||
);
|
||||
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
..
|
||||
})) => {
|
||||
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
|
||||
assert_eq!(message.text(), "Hi!");
|
||||
assert_eq!(output_token_count, 3);
|
||||
assert_eq!(usage.output_token_count, 3);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
}
|
||||
@@ -1066,11 +1058,11 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
|
||||
assert_eq!(message.message.text(), "outer");
|
||||
assert_eq!(message.finish_reason, FinishReason::Length);
|
||||
assert_eq!(
|
||||
message.prompt_token_count,
|
||||
message.usage.prompt_token_count,
|
||||
"system: You are terse.\nuser: Say hi\nassistant:".len()
|
||||
);
|
||||
assert_eq!(
|
||||
message.output_token_count,
|
||||
message.usage.output_token_count,
|
||||
"<think>inner</think>outer".len()
|
||||
);
|
||||
|
||||
|
||||
@@ -508,8 +508,11 @@ fn decoded_completion_stream(
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -519,8 +522,11 @@ fn decoded_completion_stream(
|
||||
let last_index = chunks.len() - 1;
|
||||
for (index, chunk) in chunks.into_iter().enumerate() {
|
||||
let finished = (index == last_index).then(|| Finished {
|
||||
prompt_token_count,
|
||||
output_token_count: completion_body.chars().count(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count,
|
||||
output_token_count: completion_body.chars().count(),
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user