[Rust Frontend] Stamp arrival_time at the frontend entry (#47787)
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>
This commit is contained in:
@@ -132,6 +132,10 @@ impl TextLlm {
|
||||
) -> Result<(TextRequest, GenerateOutputStream)> {
|
||||
request.validate()?;
|
||||
|
||||
if request.arrival_time.is_none() {
|
||||
request.arrival_time = Some(vllm_llm::current_unix_timestamp_secs());
|
||||
}
|
||||
|
||||
let tokenizer = self.backend.tokenizer();
|
||||
let prompt_token_ids = match take(&mut request.prompt) {
|
||||
Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
|
||||
|
||||
@@ -56,7 +56,7 @@ pub fn lower_text_request(
|
||||
data_parallel_rank: request.data_parallel_rank,
|
||||
reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
|
||||
lora_request: request.lora_request.clone(),
|
||||
arrival_time: None,
|
||||
arrival_time: request.arrival_time,
|
||||
trace_headers: None,
|
||||
};
|
||||
|
||||
@@ -1143,6 +1143,44 @@ mod tests {
|
||||
assert_eq!(prepared.generate_request.request_id, "text-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_passes_arrival_time_through() {
|
||||
let request = TextRequest {
|
||||
arrival_time: Some(42.5),
|
||||
..sample_request()
|
||||
};
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(prepared.generate_request.arrival_time, Some(42.5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_leaves_arrival_time_unset_when_absent() {
|
||||
let request = TextRequest {
|
||||
arrival_time: None,
|
||||
..sample_request()
|
||||
};
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(prepared.generate_request.arrival_time, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_max_tokens_user_smaller_than_model_limit() {
|
||||
let result = resolve_max_tokens(Some(50), None, 200, 100);
|
||||
|
||||
@@ -187,6 +187,12 @@ pub struct TextRequest {
|
||||
/// LoRA adapter selected for this request.
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<LoraRequest>,
|
||||
/// Wall-clock unix timestamp (seconds) when this request arrived at the
|
||||
/// frontend, stamped before render/tokenize to match Python's
|
||||
/// renderer-entry arrival_time. When unset, it is stamped before
|
||||
/// tokenization.
|
||||
#[serde(default)]
|
||||
pub arrival_time: Option<f64>,
|
||||
}
|
||||
|
||||
impl TextRequest {
|
||||
@@ -205,6 +211,7 @@ impl TextRequest {
|
||||
data_parallel_rank: None,
|
||||
reasoning_parser_kwargs: None,
|
||||
lora_request: None,
|
||||
arrival_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user