[Rust Frontend] Populate cached_token_count in responses (#44887)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
|
||||
use tracing::{Level, debug, trace};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
use vllm_llm::{FinishReason, GenerateOutput};
|
||||
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
|
||||
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
|
||||
|
||||
use super::logprobs::{
|
||||
@@ -40,8 +40,7 @@ impl Default for TextDecodeOptions {
|
||||
/// Terminal metadata carried on the final [`DecodedTextEvent`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Finished {
|
||||
pub prompt_token_count: usize,
|
||||
pub output_token_count: usize,
|
||||
pub usage: TokenUsage,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
@@ -98,12 +97,14 @@ pub async fn decoded_text_event_stream(
|
||||
) -> crate::Result<()> {
|
||||
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
|
||||
let mut prompt_token_count = 0_usize;
|
||||
let mut cached_token_count = 0_usize;
|
||||
let mut token_ids = Vec::new();
|
||||
let mut output_token_count: usize = 0;
|
||||
let mut logprobs: Option<DecodedLogprobs> = None;
|
||||
|
||||
while let Some(next) = raw_stream.next().await {
|
||||
let output = next?;
|
||||
cached_token_count = cached_token_count.max(output.cached_token_count);
|
||||
|
||||
// If it's the first output, init states and yield `Start` event.
|
||||
if decoder.is_none() {
|
||||
@@ -267,8 +268,11 @@ pub async fn decoded_text_event_stream(
|
||||
token_ids,
|
||||
logprobs,
|
||||
finished: Some(Finished {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
usage: TokenUsage {
|
||||
prompt_token_count,
|
||||
output_token_count,
|
||||
cached_token_count,
|
||||
},
|
||||
finish_reason: reason,
|
||||
kv_transfer_params,
|
||||
}),
|
||||
|
||||
@@ -23,6 +23,7 @@ pub struct CollectedTextOutput {
|
||||
pub logprobs: Option<DecodedLogprobs>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub finish_reason: FinishReason,
|
||||
pub usage: vllm_llm::TokenUsage,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
@@ -74,6 +75,7 @@ impl<T: TextOutputStream> T {
|
||||
logprobs: delta_logprobs,
|
||||
token_ids: delta_token_ids,
|
||||
finish_reason: FinishReason::Error,
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
kv_transfer_params: None,
|
||||
})
|
||||
};
|
||||
@@ -81,6 +83,7 @@ impl<T: TextOutputStream> T {
|
||||
if let Some(finished) = finished {
|
||||
let mut collected = collected.unwrap();
|
||||
collected.finish_reason = finished.finish_reason;
|
||||
collected.usage = finished.usage;
|
||||
collected.kv_transfer_params = finished.kv_transfer_params;
|
||||
return Ok(collected);
|
||||
}
|
||||
@@ -146,8 +149,11 @@ mod tests {
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 2,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -260,8 +266,11 @@ mod tests {
|
||||
],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 5,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 5,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user