[Rust Frontend] Populate cached_token_count in responses (#44887)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-10 20:50:05 -07:00
committed by GitHub
parent 85a0ffae42
commit 5d5591d99b
34 changed files with 556 additions and 262 deletions
+10 -18
View File
@@ -494,12 +494,12 @@ async fn chat_streams_text_events() {
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
usage,
finish_reason,
..
})) => {
assert_eq!(message.text(), "Hi");
assert_eq!(output_token_count, 3);
assert_eq!(usage.output_token_count, 3);
assert_eq!(
finish_reason,
FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))
@@ -590,13 +590,9 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
);
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
..
})) => {
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
assert_eq!(message.text(), "你");
assert_eq!(output_token_count, 4);
assert_eq!(usage.output_token_count, 4);
}
other => panic!("unexpected final event: {other:?}"),
}
@@ -681,12 +677,12 @@ async fn chat_stream_flushes_held_text_on_finish() {
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
usage,
finish_reason,
..
})) => {
assert_eq!(message.text(), "ok st");
assert_eq!(output_token_count, 5);
assert_eq!(usage.output_token_count, 5);
assert_eq!(finish_reason, FinishReason::Length);
}
other => panic!("unexpected final event: {other:?}"),
@@ -857,13 +853,9 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
);
match next_semantic(&mut stream).await {
Some(Ok(ChatEvent::Done {
message,
output_token_count,
..
})) => {
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
assert_eq!(message.text(), "Hi!");
assert_eq!(output_token_count, 3);
assert_eq!(usage.output_token_count, 3);
}
other => panic!("unexpected final event: {other:?}"),
}
@@ -1066,11 +1058,11 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
assert_eq!(message.message.text(), "outer");
assert_eq!(message.finish_reason, FinishReason::Length);
assert_eq!(
message.prompt_token_count,
message.usage.prompt_token_count,
"system: You are terse.\nuser: Say hi\nassistant:".len()
);
assert_eq!(
message.output_token_count,
message.usage.output_token_count,
"<think>inner</think>outer".len()
);
+10 -4
View File
@@ -508,8 +508,11 @@ fn decoded_completion_stream(
token_ids: Vec::new(),
logprobs: None,
finished: Some(Finished {
prompt_token_count: 0,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -519,8 +522,11 @@ fn decoded_completion_stream(
let last_index = chunks.len() - 1;
for (index, chunk) in chunks.into_iter().enumerate() {
let finished = (index == last_index).then(|| Finished {
prompt_token_count,
output_token_count: completion_body.chars().count(),
usage: vllm_llm::TokenUsage {
prompt_token_count,
output_token_count: completion_body.chars().count(),
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
});