[Rust Frontend] Populate cached_token_count in responses (#44887)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-10 20:50:05 -07:00
committed by GitHub
parent 85a0ffae42
commit 5d5591d99b
34 changed files with 556 additions and 262 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ mod request_metrics;
pub use error::{Error, Result};
pub use output::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStream,
GenerateOutputStreamExt, GeneratePromptInfo,
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage,
};
pub use request::GenerateRequest;
pub use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
+33
View File
@@ -14,6 +14,17 @@ use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
use crate::error::Result;
use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs};
/// Token usage metadata for one request.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct TokenUsage {
/// Number of prompt tokens sent to the engine.
pub prompt_token_count: usize,
/// Number of output tokens generated.
pub output_token_count: usize,
/// Number of prompt tokens served from cache.
pub cached_token_count: usize,
}
/// Final raw token output plus terminal stream metadata.
#[derive(Debug, Clone, PartialEq)]
pub struct CollectedGenerateOutput {
@@ -23,6 +34,7 @@ pub struct CollectedGenerateOutput {
pub token_ids: Vec<u32>,
pub logprobs: Option<Logprobs>,
pub finish_reason: FinishReason,
pub usage: TokenUsage,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
@@ -127,6 +139,8 @@ pub struct GenerateOutput {
pub logprobs: Option<Logprobs>,
/// Terminal finish reason, when this is the final output for the request.
pub finish_reason: Option<FinishReason>,
/// Number of prompt tokens served from cache, when reported by prefill stats.
pub cached_token_count: usize,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
@@ -173,6 +187,7 @@ impl GenerateOutput {
token_ids,
logprobs: None,
finish_reason,
cached_token_count: 0,
kv_transfer_params: None,
}
}
@@ -241,6 +256,11 @@ impl Stream for GenerateOutputStream {
}
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
let cached_token_count = raw
.prefill_stats
.as_ref()
.map(|stats| stats.num_cached_tokens as usize)
.unwrap_or(0);
let finish_reason = finish_reason_from_engine(raw.finish_reason, raw.stop_reason);
if let Some(finish_reason) = finish_reason.as_ref() {
@@ -253,6 +273,7 @@ impl Stream for GenerateOutputStream {
token_ids: raw.new_token_ids,
logprobs,
finish_reason,
cached_token_count,
kv_transfer_params: raw.kv_transfer_params,
};
@@ -299,9 +320,11 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
pin_mut!(stream);
let mut prompt_token_ids = None;
let mut prompt_logprobs = None;
let mut cached_token_count = 0;
let mut collected: Option<CollectedGenerateOutput> = None;
while let Some(output) = stream.next().await.transpose()? {
cached_token_count = cached_token_count.max(output.cached_token_count);
if let Some(info) = output.prompt_info {
if prompt_token_ids.is_none() {
prompt_token_ids = Some(info.prompt_token_ids.to_vec());
@@ -328,6 +351,11 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
token_ids: output.token_ids,
logprobs: output.logprobs,
finish_reason: FinishReason::Error,
usage: TokenUsage {
prompt_token_count: prompt_token_ids.as_ref().map_or(0, Vec::len),
output_token_count: 0,
cached_token_count,
},
kv_transfer_params: None,
});
}
@@ -335,6 +363,11 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
if let Some(finish_reason) = output.finish_reason {
let mut collected = collected.expect("terminal output must exist");
collected.finish_reason = finish_reason;
collected.usage = TokenUsage {
prompt_token_count: collected.prompt_token_ids.len(),
output_token_count: collected.token_ids.len(),
cached_token_count,
};
collected.kv_transfer_params = output.kv_transfer_params;
return Ok(collected);
}
+16 -7
View File
@@ -332,13 +332,21 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
EngineCoreOutputs {
engine_index: 0,
outputs: vec![
request_output_with_logprobs(
&request.request_id,
vec![33],
None,
Some(logprobs_for_position(33, -0.1, 1, 99, -0.2)),
Some(prompt_logprobs()),
),
EngineCoreOutput {
prefill_stats: Some(PrefillStats {
num_prompt_tokens: 2,
num_cached_tokens: 1,
num_local_cached_tokens: 1,
..Default::default()
}),
..request_output_with_logprobs(
&request.request_id,
vec![33],
None,
Some(logprobs_for_position(33, -0.1, 1, 99, -0.2)),
Some(prompt_logprobs()),
)
},
request_output_with_logprobs_and_kv(
&request.request_id,
vec![44],
@@ -373,6 +381,7 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
assert_eq!(collected.prompt_token_ids, vec![11, 22]);
assert_eq!(collected.token_ids, vec![33, 44]);
assert_eq!(collected.finish_reason, FinishReason::stop_eos());
assert_eq!(collected.usage.cached_token_count, 1);
assert_eq!(collected.prompt_logprobs, Some(prompt_logprobs()));
assert_eq!(
collected.logprobs.as_ref().map(|lp| lp.positions.len()),