[Rust Frontend] Populate cached_token_count in responses (#44887)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
+10
-18
@@ -494,12 +494,12 @@ async fn chat_streams_text_events() {
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
..
|
||||
})) => {
|
||||
assert_eq!(message.text(), "Hi");
|
||||
assert_eq!(output_token_count, 3);
|
||||
assert_eq!(usage.output_token_count, 3);
|
||||
assert_eq!(
|
||||
finish_reason,
|
||||
FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))
|
||||
@@ -590,13 +590,9 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
|
||||
);
|
||||
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
..
|
||||
})) => {
|
||||
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
|
||||
assert_eq!(message.text(), "你");
|
||||
assert_eq!(output_token_count, 4);
|
||||
assert_eq!(usage.output_token_count, 4);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
}
|
||||
@@ -681,12 +677,12 @@ async fn chat_stream_flushes_held_text_on_finish() {
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
..
|
||||
})) => {
|
||||
assert_eq!(message.text(), "ok st");
|
||||
assert_eq!(output_token_count, 5);
|
||||
assert_eq!(usage.output_token_count, 5);
|
||||
assert_eq!(finish_reason, FinishReason::Length);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
@@ -857,13 +853,9 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
|
||||
);
|
||||
|
||||
match next_semantic(&mut stream).await {
|
||||
Some(Ok(ChatEvent::Done {
|
||||
message,
|
||||
output_token_count,
|
||||
..
|
||||
})) => {
|
||||
Some(Ok(ChatEvent::Done { message, usage, .. })) => {
|
||||
assert_eq!(message.text(), "Hi!");
|
||||
assert_eq!(output_token_count, 3);
|
||||
assert_eq!(usage.output_token_count, 3);
|
||||
}
|
||||
other => panic!("unexpected final event: {other:?}"),
|
||||
}
|
||||
@@ -1066,11 +1058,11 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
|
||||
assert_eq!(message.message.text(), "outer");
|
||||
assert_eq!(message.finish_reason, FinishReason::Length);
|
||||
assert_eq!(
|
||||
message.prompt_token_count,
|
||||
message.usage.prompt_token_count,
|
||||
"system: You are terse.\nuser: Say hi\nassistant:".len()
|
||||
);
|
||||
assert_eq!(
|
||||
message.output_token_count,
|
||||
message.usage.output_token_count,
|
||||
"<think>inner</think>outer".len()
|
||||
);
|
||||
|
||||
|
||||
@@ -508,8 +508,11 @@ fn decoded_completion_stream(
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -519,8 +522,11 @@ fn decoded_completion_stream(
|
||||
let last_index = chunks.len() - 1;
|
||||
for (index, chunk) in chunks.into_iter().enumerate() {
|
||||
let finished = (index == last_index).then(|| Finished {
|
||||
prompt_token_count,
|
||||
output_token_count: completion_body.chars().count(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count,
|
||||
output_token_count: completion_body.chars().count(),
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user