[Rust Frontend] Add per-request timing metrics to chat and completions (#59314)
Signed-off-by: wxin <wxin@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
802feb6770
commit
1bc6e072b0
@@ -10,7 +10,7 @@ use tracing::{Level, debug, trace};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
|
||||
use vllm_llm::{FinishReason, GenerateOutput, RequestTimestamps, TokenUsage};
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder};
|
||||
|
||||
use super::logprobs::{
|
||||
@@ -55,6 +55,7 @@ pub struct Finished {
|
||||
pub ec_transfer_params: Option<Box<serde_json::Value>>,
|
||||
/// Sampling support sets aligned with all generated token positions.
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
pub timestamps: RequestTimestamps,
|
||||
}
|
||||
|
||||
/// Sample metadata emitted by one engine output update.
|
||||
@@ -317,6 +318,7 @@ pub async fn decoded_text_event_stream(
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
sampling_mask,
|
||||
timestamps: output.timestamps,
|
||||
})),
|
||||
})
|
||||
.await;
|
||||
@@ -543,7 +545,19 @@ mod tests {
|
||||
name: "utf8 byte fallback resolves across engine updates",
|
||||
outputs: vec![
|
||||
GenerateOutput::for_test(Some(Arc::clone(&prompt)), vec![0xe4], None),
|
||||
GenerateOutput::for_test(None, vec![0xbd, 0xa0], Some(FinishReason::Length)),
|
||||
GenerateOutput {
|
||||
timestamps: RequestTimestamps {
|
||||
queued_ts: 8.0,
|
||||
scheduled_ts: 9.0,
|
||||
first_token_ts: 10.0,
|
||||
last_token_ts: 11.5,
|
||||
},
|
||||
..GenerateOutput::for_test(
|
||||
None,
|
||||
vec![0xbd, 0xa0],
|
||||
Some(FinishReason::Length),
|
||||
)
|
||||
},
|
||||
],
|
||||
expected: vec![
|
||||
DecodedTextEvent::Start {
|
||||
@@ -565,6 +579,12 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
timestamps: RequestTimestamps {
|
||||
queued_ts: 8.0,
|
||||
scheduled_ts: 9.0,
|
||||
first_token_ts: 10.0,
|
||||
last_token_ts: 11.5,
|
||||
},
|
||||
}),
|
||||
),
|
||||
],
|
||||
@@ -597,6 +617,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
timestamps: Default::default(),
|
||||
}),
|
||||
),
|
||||
],
|
||||
@@ -634,12 +655,19 @@ mod tests {
|
||||
async fn stream_stop_string_sets_task_local_abort_cause_on_raw_stream_drop() {
|
||||
let prompt: Arc<[u32]> = Arc::from([]);
|
||||
let dropped_cause = Arc::new(Mutex::new(None));
|
||||
let timestamps = RequestTimestamps {
|
||||
queued_ts: 1.0,
|
||||
scheduled_ts: 2.0,
|
||||
first_token_ts: 3.0,
|
||||
last_token_ts: 4.0,
|
||||
};
|
||||
let raw_stream = DropRecordingStream {
|
||||
next: Some(Ok(GenerateOutput::for_test(
|
||||
Some(prompt),
|
||||
ascii_tokens("hello"),
|
||||
Some(FinishReason::Length),
|
||||
))),
|
||||
// The engine output is not terminal. The frontend terminates the
|
||||
// request after decoding the configured stop string.
|
||||
next: Some(Ok(GenerateOutput {
|
||||
timestamps,
|
||||
..GenerateOutput::for_test(Some(prompt), ascii_tokens("hello"), None)
|
||||
})),
|
||||
dropped_cause: Arc::clone(&dropped_cause),
|
||||
};
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
@@ -657,6 +685,7 @@ mod tests {
|
||||
|
||||
assert_eq!(output.text, "he");
|
||||
assert!(output.finish_reason.is_stop());
|
||||
assert_eq!(output.timestamps, timestamps);
|
||||
assert_eq!(
|
||||
*dropped_cause.lock().unwrap(),
|
||||
Some(AbortCause::StopStringMatched)
|
||||
|
||||
@@ -31,6 +31,7 @@ pub struct CollectedTextOutput {
|
||||
pub token_ids: Vec<u32>,
|
||||
pub finish_reason: FinishReason,
|
||||
pub usage: vllm_llm::TokenUsage,
|
||||
pub timestamps: vllm_llm::RequestTimestamps,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
/// Connector-specific encoder cache transfer parameters for disaggregated
|
||||
@@ -91,6 +92,7 @@ impl<T: TextOutputStream> T {
|
||||
token_ids: delta_token_ids,
|
||||
finish_reason: FinishReason::Error,
|
||||
usage: vllm_llm::TokenUsage::default(),
|
||||
timestamps: vllm_llm::RequestTimestamps::default(),
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
@@ -101,6 +103,7 @@ impl<T: TextOutputStream> T {
|
||||
let mut collected = collected.unwrap();
|
||||
collected.finish_reason = finished.finish_reason;
|
||||
collected.usage = finished.usage;
|
||||
collected.timestamps = finished.timestamps;
|
||||
collected.kv_transfer_params =
|
||||
finished.kv_transfer_params.map(|value| *value);
|
||||
collected.ec_transfer_params =
|
||||
@@ -181,6 +184,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
timestamps: Default::default(),
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
@@ -304,6 +308,7 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
timestamps: Default::default(),
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user