[Rust Frontend] Populate cached_token_count in responses (#44887)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-10 20:50:05 -07:00
committed by GitHub
parent 85a0ffae42
commit 5d5591d99b
34 changed files with 556 additions and 262 deletions
+9 -5
View File
@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
use tracing::{Level, debug, trace};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::StopReason;
use vllm_llm::{FinishReason, GenerateOutput};
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
use super::logprobs::{
@@ -40,8 +40,7 @@ impl Default for TextDecodeOptions {
/// Terminal metadata carried on the final [`DecodedTextEvent`].
#[derive(Debug, Clone, PartialEq)]
pub struct Finished {
pub prompt_token_count: usize,
pub output_token_count: usize,
pub usage: TokenUsage,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
@@ -98,12 +97,14 @@ pub async fn decoded_text_event_stream(
) -> crate::Result<()> {
let mut decoder: Option<Box<dyn IncrementalDecoder>> = None;
let mut prompt_token_count = 0_usize;
let mut cached_token_count = 0_usize;
let mut token_ids = Vec::new();
let mut output_token_count: usize = 0;
let mut logprobs: Option<DecodedLogprobs> = None;
while let Some(next) = raw_stream.next().await {
let output = next?;
cached_token_count = cached_token_count.max(output.cached_token_count);
// If it's the first output, init states and yield `Start` event.
if decoder.is_none() {
@@ -267,8 +268,11 @@ pub async fn decoded_text_event_stream(
token_ids,
logprobs,
finished: Some(Finished {
prompt_token_count,
output_token_count,
usage: TokenUsage {
prompt_token_count,
output_token_count,
cached_token_count,
},
finish_reason: reason,
kv_transfer_params,
}),
+13 -4
View File
@@ -23,6 +23,7 @@ pub struct CollectedTextOutput {
pub logprobs: Option<DecodedLogprobs>,
pub token_ids: Vec<u32>,
pub finish_reason: FinishReason,
pub usage: vllm_llm::TokenUsage,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
@@ -74,6 +75,7 @@ impl<T: TextOutputStream> T {
logprobs: delta_logprobs,
token_ids: delta_token_ids,
finish_reason: FinishReason::Error,
usage: vllm_llm::TokenUsage::default(),
kv_transfer_params: None,
})
};
@@ -81,6 +83,7 @@ impl<T: TextOutputStream> T {
if let Some(finished) = finished {
let mut collected = collected.unwrap();
collected.finish_reason = finished.finish_reason;
collected.usage = finished.usage;
collected.kv_transfer_params = finished.kv_transfer_params;
return Ok(collected);
}
@@ -146,8 +149,11 @@ mod tests {
],
}),
finished: Some(Finished {
prompt_token_count: 2,
output_token_count: 2,
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 2,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -260,8 +266,11 @@ mod tests {
],
}),
finished: Some(Finished {
prompt_token_count: 2,
output_token_count: 5,
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 5,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),