[Rust Frontend] Stamp arrival_time at the frontend entry (#47787)
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>
This commit is contained in:
@@ -14,6 +14,7 @@ pub use output::{
|
||||
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage,
|
||||
};
|
||||
pub use request::GenerateRequest;
|
||||
pub use request_metrics::current_unix_timestamp_secs;
|
||||
pub use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
|
||||
|
||||
use crate::inflight::InflightRequests;
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
@@ -8,6 +7,7 @@ use vllm_engine_core_client::protocol::request::{EngineCoreRequest, ReasoningPar
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request_metrics::current_unix_timestamp_secs;
|
||||
|
||||
/// Tokenized decoder-only generate request accepted by [`crate::Llm`].
|
||||
///
|
||||
@@ -30,8 +30,9 @@ pub struct GenerateRequest {
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
/// Unix timestamp, in seconds, when this request arrived at the frontend.
|
||||
///
|
||||
/// When omitted, the Rust frontend fills it immediately before sending the
|
||||
/// request to engine-core, matching Python's default arrival-time behavior.
|
||||
/// Stamped at the frontend entry, before render and tokenization, to match
|
||||
/// Python's renderer-entry arrival_time. When omitted, it is filled as a
|
||||
/// fallback before the request is sent to engine-core.
|
||||
pub arrival_time: Option<f64>,
|
||||
/// Optional salt used to partition prefix-cache entries for this request.
|
||||
pub cache_salt: Option<String>,
|
||||
@@ -122,13 +123,6 @@ impl PreparedGenerateRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn current_unix_timestamp_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock is before unix epoch")
|
||||
.as_secs_f64()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -330,7 +330,7 @@ fn diff_or_zero(end: f64, start: f64) -> f64 {
|
||||
///
|
||||
/// Original Python request timestamp source:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L206-L216>
|
||||
pub(crate) fn current_unix_timestamp_secs() -> f64 {
|
||||
pub fn current_unix_timestamp_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock is before unix epoch")
|
||||
|
||||
Reference in New Issue
Block a user