[Rust Frontend] Add iteration token histogram (#56990)

Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-15 12:12:15 +00:00
committed by GitHub
co-authored by Codex
parent bb55077411
commit 78eaaad2c7
5 changed files with 90 additions and 14 deletions
+14 -5
View File
@@ -19,6 +19,7 @@ use vllm_engine_core_client::protocol::output::{
UtilityCallOutput,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::protocol::utility::{
EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope,
};
@@ -187,6 +188,11 @@ impl ActiveRequest {
/// Advance this request by one mock engine step.
fn step(&mut self, opt: &Opt) -> EngineCoreOutput {
let prefill_stats = (self.generated == 0).then(|| PrefillStats {
num_prompt_tokens: self.prompt_len as u32,
num_computed_tokens: self.prompt_len as u32,
..Default::default()
});
let remaining = self.max_tokens - self.generated;
let chunk_len = remaining.min(opt.output_token_chunk_size);
let mut new_token_ids = Vec::with_capacity(chunk_len);
@@ -196,11 +202,14 @@ impl ActiveRequest {
self.generated += chunk_len;
let finished = self.generated >= self.max_tokens;
request_output(
self.request_id.clone(),
new_token_ids,
finished.then_some(EngineCoreFinishReason::Length),
)
EngineCoreOutput {
prefill_stats,
..request_output(
self.request_id.clone(),
new_token_ids,
finished.then_some(EngineCoreFinishReason::Length),
)
}
}
}
+11
View File
@@ -10,6 +10,7 @@ use tokio_util::sync::CancellationToken;
use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::test_utils::IpcNamespace;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
@@ -119,12 +120,22 @@ async fn chunk_size_one_outputs_one_token_per_update() {
let first = stream.next().await.expect("first").expect("first ok");
assert_eq!(first.new_token_ids.len(), 1);
assert_eq!(first.finish_reason, None);
assert_eq!(
first.prefill_stats,
Some(PrefillStats {
num_prompt_tokens: 3,
num_computed_tokens: 3,
..Default::default()
})
);
let second = stream.next().await.expect("second").expect("second ok");
assert_eq!(second.new_token_ids.len(), 1);
assert_eq!(second.finish_reason, None);
assert!(second.prefill_stats.is_none());
let third = stream.next().await.expect("third").expect("third ok");
assert_eq!(third.new_token_ids.len(), 1);
assert_eq!(third.finish_reason, Some(EngineCoreFinishReason::Length));
assert!(third.prefill_stats.is_none());
assert!(stream.next().await.is_none());
shutdown_mock(client, shutdown, task).await;