From aabc1a0a0b55c768ab1f8962157f5fa086932b88 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Tue, 18 Aug 2026 22:52:25 -0400 Subject: [PATCH] [Bugfix][Benchmark] Check readiness before tokenizer init in rust vllm-bench (#51863) Signed-off-by: Tyler Michael Smith Co-authored-by: OpenAI Codex --- rust/src/bench/src/benchmark.rs | 51 +++--------- rust/src/bench/src/multi_turn.rs | 50 ++++------- rust/src/bench/src/ready_checker.rs | 124 +++++++++++++++++++++++++++- rust/src/bench/src/tokenizer.rs | 15 ++-- 4 files changed, 159 insertions(+), 81 deletions(-) diff --git a/rust/src/bench/src/benchmark.rs b/rust/src/bench/src/benchmark.rs index cc4acc4f06..d2a7bf8480 100644 --- a/rust/src/bench/src/benchmark.rs +++ b/rust/src/bench/src/benchmark.rs @@ -18,7 +18,7 @@ use crate::metrics::steady_state; use crate::output::console::print_results; use crate::output::json::{append_result, build_result_json, compute_result_filename, save_result}; use crate::rate_control::compute_schedule; -use crate::ready_checker::wait_for_endpoint; +use crate::ready_checker::{get_first_model, wait_for_endpoint}; /// Pre-resolve the hostname in `base_url` and pin all resolved IPs on the /// client builder via [`reqwest::ClientBuilder::resolve_to_addrs`]. This @@ -281,40 +281,6 @@ pub(crate) fn assign_lora_modules( Some(out) } -/// Fetch the first model from the server's /v1/models endpoint. -async fn get_first_model_from_server( - base_url: &str, - client: &reqwest::Client, - extra_headers: &Option>, -) -> Result<(String, String)> { - let url = format!("{base_url}/v1/models"); - let mut request = client.get(&url); - if let Some(headers) = extra_headers { - for (k, v) in headers { - request = request.header(k, v); - } - } - // Add API key from environment - if let Ok(api_key) = std::env::var("OPENAI_API_KEY") { - request = request.header("Authorization", format!("Bearer {api_key}")); - } - - let response = request.send().await?; - let data: serde_json::Value = response.json().await?; - - if let Some(models) = data.get("data").and_then(|d| d.as_array()) - && let Some(first) = models.first() - { - let id = first.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string(); - let root = first.get("root").and_then(|v| v.as_str()).unwrap_or(&id).to_string(); - return Ok((id, root)); - } - - Err(BenchError::Config(format!( - "No models found on the server at {base_url}" - ))) -} - /// Run the complete benchmark. /// /// This is the core orchestrator matching Python's benchmark() + main_async(). @@ -348,8 +314,13 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result { (m.clone(), config.model_name.clone()) } else { tracing::info!(base_url = %config.base_url, "fetching first model from server"); - let (name, id) = - get_first_model_from_server(&config.base_url, &client, &config.extra_headers).await?; + let (name, id) = get_first_model( + &config.base_url, + &client, + &config.extra_headers, + config.ready_check_timeout_sec, + ) + .await?; tracing::info!( model_name = name, model_id = id, @@ -364,7 +335,11 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result { } else { let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id); tracing::info!(tokenizer = tid, "loading tokenizer"); - let server_info = Some((config.base_url.as_str(), model_id.as_str())); + let server_info = Some(( + config.base_url.as_str(), + model_id.as_str(), + config.ready_check_timeout_sec, + )); let t = crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info).await?; Some(t) diff --git a/rust/src/bench/src/multi_turn.rs b/rust/src/bench/src/multi_turn.rs index 4b962b879c..e0a3138170 100644 --- a/rust/src/bench/src/multi_turn.rs +++ b/rust/src/bench/src/multi_turn.rs @@ -23,6 +23,7 @@ use crate::output::console::print_multi_turn_results; use crate::output::json::{ append_result, build_multi_turn_result_json, compute_result_filename, save_result, }; +use crate::ready_checker::{get_first_model, wait_for_endpoint}; /// Output from a single turn within a conversation. #[derive(Debug, Clone)] @@ -74,7 +75,13 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result Result Result>, -) -> Result<(String, String)> { - let url = format!("{base_url}/v1/models"); - let mut request = client.get(&url); - if let Some(headers) = extra_headers { - for (k, v) in headers { - request = request.header(k, v); - } - } - if let Ok(api_key) = std::env::var("OPENAI_API_KEY") { - request = request.header("Authorization", format!("Bearer {api_key}")); - } - - let response = request.send().await?; - let data: serde_json::Value = response.json().await?; - - if let Some(models) = data.get("data").and_then(|d| d.as_array()) - && let Some(first) = models.first() - { - let id = first.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string(); - let root = first.get("root").and_then(|v| v.as_str()).unwrap_or(&id).to_string(); - return Ok((id, root)); - } - - Err(BenchError::Config(format!( - "No models found on the server at {base_url}" - ))) -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/rust/src/bench/src/ready_checker.rs b/rust/src/bench/src/ready_checker.rs index 144a1ebd99..77d35fb278 100644 --- a/rust/src/bench/src/ready_checker.rs +++ b/rust/src/bench/src/ready_checker.rs @@ -1,14 +1,96 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::future::Future; use std::time::Instant; use indicatif::{ProgressBar, ProgressStyle}; +use thiserror_ext::AsReport as _; use crate::backends::{RequestFuncInput, RequestFuncOutput, get_backend}; use crate::cli::BackendKind; use crate::error::{BenchError, Result}; +/// Retry an operation until it succeeds or the readiness timeout expires. +pub(crate) async fn retry_with_timeout( + timeout_seconds: u64, + retry_interval: u64, + mut operation: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + if timeout_seconds == 0 { + return operation().await; + } + + let deadline = Instant::now() + std::time::Duration::from_secs(timeout_seconds); + + loop { + let error = match operation().await { + Ok(value) => return Ok(value), + Err(error) => error, + }; + tracing::warn!(error = %error.as_report(), "endpoint is not ready"); + + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(BenchError::EndpointTimeout( + timeout_seconds, + format!("{}", error.as_report()), + )); + } + + let sleep_duration = + std::cmp::min(std::time::Duration::from_secs(retry_interval), remaining); + if !sleep_duration.is_zero() { + tokio::time::sleep(sleep_duration).await; + } + } +} + +/// Fetch the first model from the server, retrying while it starts. +pub(crate) async fn get_first_model( + base_url: &str, + client: &reqwest::Client, + extra_headers: &Option>, + timeout_seconds: u64, +) -> Result<(String, String)> { + let url = format!("{base_url}/v1/models"); + retry_with_timeout(timeout_seconds, 5, || async { + let mut request = client.get(&url); + if let Some(headers) = extra_headers { + for (key, value) in headers { + request = request.header(key, value); + } + } + // Add API key from environment + if let Ok(api_key) = std::env::var("OPENAI_API_KEY") { + request = request.header("Authorization", format!("Bearer {api_key}")); + } + + let response = request.send().await?.error_for_status()?; + let data: serde_json::Value = response.json().await?; + if let Some(model) = data + .get("data") + .and_then(|value| value.as_array()) + .and_then(|models| models.first()) + { + let id = + model.get("id").and_then(|value| value.as_str()).unwrap_or_default().to_string(); + let root = + model.get("root").and_then(|value| value.as_str()).unwrap_or(&id).to_string(); + return Ok((id, root)); + } + + Err(BenchError::Config(format!( + "No models found on the server at {base_url}" + ))) + }) + .await +} + /// Wait for the serving endpoint to become available. /// /// Sends test requests with retry until success or timeout. @@ -63,7 +145,7 @@ pub async fn wait_for_endpoint( last_error = err; } Err(e) => { - last_error = e.to_string(); + last_error = format!("{}", e.as_report()); } } @@ -76,3 +158,43 @@ pub async fn wait_for_endpoint( Err(BenchError::EndpointTimeout(timeout_seconds, last_error)) } + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + #[tokio::test] + async fn retry_succeeds_after_transient_failures() { + let attempts = AtomicUsize::new(0); + + let value = retry_with_timeout(1, 0, || async { + if attempts.fetch_add(1, Ordering::Relaxed) < 2 { + Err(BenchError::Backend("not ready".into())) + } else { + Ok(42) + } + }) + .await + .unwrap(); + + assert_eq!(value, 42); + assert_eq!(attempts.load(Ordering::Relaxed), 3); + } + + #[tokio::test] + async fn zero_timeout_does_not_retry() { + let attempts = AtomicUsize::new(0); + + let error = retry_with_timeout(0, 0, || async { + attempts.fetch_add(1, Ordering::Relaxed); + Err::<(), _>(BenchError::Backend("not ready".into())) + }) + .await + .unwrap_err(); + + assert!(matches!(error, BenchError::Backend(_))); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + } +} diff --git a/rust/src/bench/src/tokenizer.rs b/rust/src/bench/src/tokenizer.rs index add890c391..9854386e9a 100644 --- a/rust/src/bench/src/tokenizer.rs +++ b/rust/src/bench/src/tokenizer.rs @@ -30,7 +30,7 @@ pub struct ServerTokenizer { impl ServerTokenizer { /// Create a new server tokenizer and verify connectivity. - pub async fn new(base_url: &str, model: &str) -> Result { + pub async fn new(base_url: &str, model: &str, timeout_seconds: u64) -> Result { let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(30)) .build() @@ -49,7 +49,10 @@ impl ServerTokenizer { }; // Probe the endpoint to verify it works and discover vocab size - let test_tokens = st.encode_async("test").await?; + let test_tokens = crate::ready_checker::retry_with_timeout(timeout_seconds, 5, || { + st.encode_async("test") + }) + .await?; let max_id = test_tokens.iter().copied().max().unwrap_or(0); let estimated_vocab = (max_id * 2).max(131072); @@ -225,11 +228,11 @@ impl TokenizerKind { /// 2. Tiktoken model file (for Kimi, Qwen, etc.) /// 3. Server-side /tokenize + /detokenize endpoints /// -/// `server_info` is `Some((base_url, model))` to enable server-side fallback. +/// `server_info` is `Some((base_url, model, timeout_seconds))` to enable server-side fallback. pub async fn load_tokenizer( model_id: &str, _trust_remote_code: bool, - server_info: Option<(&str, &str)>, + server_info: Option<(&str, &str, u64)>, ) -> Result { // 0. Check for built-in tiktoken encoding names (no HF download needed). These are useful for // consistent cross-model token counting (e.g. Artificial Analysis). @@ -275,13 +278,13 @@ pub async fn load_tokenizer( } Err(tiktoken_err) => { // 3. Try server-side fallback - if let Some((base_url, model)) = server_info { + if let Some((base_url, model, timeout_seconds)) = server_info { tracing::info!( model = model_id, error = %tiktoken_err.as_report(), "tiktoken unavailable; trying server-side tokenization" ); - match ServerTokenizer::new(base_url, model).await { + match ServerTokenizer::new(base_url, model, timeout_seconds).await { Ok(srv) => { tracing::info!( model = model_id,