[Rust Frontend] Add per-request timing metrics to chat and completions (#59314)

Signed-off-by: wxin <wxin@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
xinnywinne
2026-10-12 15:39:10 -07:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent 802feb6770
commit 1bc6e072b0
28 changed files with 517 additions and 25 deletions
+36 -7
View File
@@ -10,7 +10,7 @@ use tracing::{Level, debug, trace};
use vllm_engine_core_client::AbortCause;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
use vllm_llm::{FinishReason, GenerateOutput, RequestTimestamps, TokenUsage};
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
use super::logprobs::{
@@ -55,6 +55,7 @@ pub struct Finished {
pub ec_transfer_params: Option<Box<serde_json::Value>>,
/// Sampling support sets aligned with all generated token positions.
pub sampling_mask: Option<SamplingMask>,
pub timestamps: RequestTimestamps,
}
/// Sample metadata emitted by one engine output update.
@@ -317,6 +318,7 @@ pub async fn decoded_text_event_stream(
kv_transfer_params,
ec_transfer_params,
sampling_mask,
timestamps: output.timestamps,
})),
})
.await;
@@ -543,7 +545,19 @@ mod tests {
name: "utf8 byte fallback resolves across engine updates",
outputs: vec![
GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None),
GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)),
GenerateOutput {
timestamps: RequestTimestamps {
queued_ts: 8.0,
scheduled_ts: 9.0,
first_token_ts: 10.0,
last_token_ts: 11.5,
},
..GenerateOutput::for_test(
None,
vec![0xbd, 0xa0],
Some(FinishReason::Length),
)
},
],
expected: vec![
DecodedTextEvent::Start {
@@ -565,6 +579,12 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
timestamps: RequestTimestamps {
queued_ts: 8.0,
scheduled_ts: 9.0,
first_token_ts: 10.0,
last_token_ts: 11.5,
},
}),
),
],
@@ -597,6 +617,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
timestamps: Default::default(),
}),
),
],
@@ -634,12 +655,19 @@ mod tests {
async fn stream_stop_string_sets_task_local_abort_cause_on_raw_stream_drop() {
let prompt: Arc<[u32]> = Arc::from([]);
let dropped_cause = Arc::new(Mutex::new(None));
let timestamps = RequestTimestamps {
queued_ts: 1.0,
scheduled_ts: 2.0,
first_token_ts: 3.0,
last_token_ts: 4.0,
};
let raw_stream = DropRecordingStream {
next: Some(Ok(GenerateOutput::for_test(
Some(prompt),
ascii_tokens("hello"),
Some(FinishReason::Length),
))),
// The engine output is not terminal. The frontend terminates the
// request after decoding the configured stop string.
next: Some(Ok(GenerateOutput {
timestamps,
..GenerateOutput::for_test(Some(prompt), ascii_tokens("hello"), None)
})),
dropped_cause: Arc::clone(&dropped_cause),
};
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
@@ -657,6 +685,7 @@ mod tests {
assert_eq!(output.text, "he");
assert!(output.finish_reason.is_stop());
assert_eq!(output.timestamps, timestamps);
assert_eq!(
*dropped_cause.lock().unwrap(),
Some(AbortCause::StopStringMatched)
+5
View File
@@ -31,6 +31,7 @@ pub struct CollectedTextOutput {
pub token_ids: Vec<u32>,
pub finish_reason: FinishReason,
pub usage: vllm_llm::TokenUsage,
pub timestamps: vllm_llm::RequestTimestamps,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
/// Connector-specific encoder cache transfer parameters for disaggregated
@@ -91,6 +92,7 @@ impl<T: TextOutputStream> T {
token_ids: delta_token_ids,
finish_reason: FinishReason::Error,
usage: vllm_llm::TokenUsage::default(),
timestamps: vllm_llm::RequestTimestamps::default(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
@@ -101,6 +103,7 @@ impl<T: TextOutputStream> T {
let mut collected = collected.unwrap();
collected.finish_reason = finished.finish_reason;
collected.usage = finished.usage;
collected.timestamps = finished.timestamps;
collected.kv_transfer_params =
finished.kv_transfer_params.map(|value| *value);
collected.ec_transfer_params =
@@ -181,6 +184,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
timestamps: Default::default(),
})),
}),
]);
@@ -304,6 +308,7 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
timestamps: Default::default(),
})),
}),
]);