[Bugfix][Benchmark] Check readiness before tokenizer init in rust vllm-bench (#51863)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tyler Michael Smith
2026-08-19 02:52:25 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 03b87dc42f
commit aabc1a0a0b
4 changed files with 159 additions and 81 deletions
+13 -38
View File
@@ -18,7 +18,7 @@ use crate::metrics::steady_state;
use crate::output::console::print_results;
use crate::output::json::{append_result, build_result_json, compute_result_filename, save_result};
use crate::rate_control::compute_schedule;
use crate::ready_checker::wait_for_endpoint;
use crate::ready_checker::{get_first_model, wait_for_endpoint};
/// Pre-resolve the hostname in `base_url` and pin all resolved IPs on the
/// client builder via [`reqwest::ClientBuilder::resolve_to_addrs`]. This
@@ -281,40 +281,6 @@ pub(crate) fn assign_lora_modules(
Some(out)
}
/// Fetch the first model from the server's /v1/models endpoint.
async fn get_first_model_from_server(
base_url: &str,
client: &reqwest::Client,
extra_headers: &Option<std::collections::HashMap<String, String>>,
) -> Result<(String, String)> {
let url = format!("{base_url}/v1/models");
let mut request = client.get(&url);
if let Some(headers) = extra_headers {
for (k, v) in headers {
request = request.header(k, v);
}
}
// Add API key from environment
if let Ok(api_key) = std::env::var("OPENAI_API_KEY") {
request = request.header("Authorization", format!("Bearer {api_key}"));
}
let response = request.send().await?;
let data: serde_json::Value = response.json().await?;
if let Some(models) = data.get("data").and_then(|d| d.as_array())
&& let Some(first) = models.first()
{
let id = first.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string();
let root = first.get("root").and_then(|v| v.as_str()).unwrap_or(&id).to_string();
return Ok((id, root));
}
Err(BenchError::Config(format!(
"No models found on the server at {base_url}"
)))
}
/// Run the complete benchmark.
///
/// This is the core orchestrator matching Python's benchmark() + main_async().
@@ -348,8 +314,13 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
(m.clone(), config.model_name.clone())
} else {
tracing::info!(base_url = %config.base_url, "fetching first model from server");
let (name, id) =
get_first_model_from_server(&config.base_url, &client, &config.extra_headers).await?;
let (name, id) = get_first_model(
&config.base_url,
&client,
&config.extra_headers,
config.ready_check_timeout_sec,
)
.await?;
tracing::info!(
model_name = name,
model_id = id,
@@ -364,7 +335,11 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
} else {
let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id);
tracing::info!(tokenizer = tid, "loading tokenizer");
let server_info = Some((config.base_url.as_str(), model_id.as_str()));
let server_info = Some((
config.base_url.as_str(),
model_id.as_str(),
config.ready_check_timeout_sec,
));
let t =
crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info).await?;
Some(t)
+14 -36
View File
@@ -23,6 +23,7 @@ use crate::output::console::print_multi_turn_results;
use crate::output::json::{
append_result, build_multi_turn_result_json, compute_result_filename, save_result,
};
use crate::ready_checker::{get_first_model, wait_for_endpoint};
/// Output from a single turn within a conversation.
#[derive(Debug, Clone)]
@@ -74,7 +75,13 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
(m.clone(), config.model_name.clone())
} else {
tracing::info!(base_url = %config.base_url, "fetching first model from server");
let (name, id) = get_first_model(&config.base_url, &client, &config.extra_headers).await?;
let (name, id) = get_first_model(
&config.base_url,
&client,
&config.extra_headers,
config.ready_check_timeout_sec,
)
.await?;
tracing::info!(
model_name = name,
model_id = id,
@@ -89,7 +96,11 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
} else {
let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id);
tracing::info!(tokenizer = tid, "loading tokenizer");
let server_info = Some((config.base_url.as_str(), model_id.as_str()));
let server_info = Some((
config.base_url.as_str(),
model_id.as_str(),
config.ready_check_timeout_sec,
));
let t =
crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info).await?;
Some(t)
@@ -263,7 +274,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
};
tracing::info!("starting initial single-prompt test run");
let test_output = crate::ready_checker::wait_for_endpoint(
let test_output = wait_for_endpoint(
config.backend,
&client,
&test_input,
@@ -748,39 +759,6 @@ async fn run_conversation(
}
}
/// Fetch the first model from the server's /v1/models endpoint.
async fn get_first_model(
base_url: &str,
client: &reqwest::Client,
extra_headers: &Option<HashMap<String, String>>,
) -> Result<(String, String)> {
let url = format!("{base_url}/v1/models");
let mut request = client.get(&url);
if let Some(headers) = extra_headers {
for (k, v) in headers {
request = request.header(k, v);
}
}
if let Ok(api_key) = std::env::var("OPENAI_API_KEY") {
request = request.header("Authorization", format!("Bearer {api_key}"));
}
let response = request.send().await?;
let data: serde_json::Value = response.json().await?;
if let Some(models) = data.get("data").and_then(|d| d.as_array())
&& let Some(first) = models.first()
{
let id = first.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string();
let root = first.get("root").and_then(|v| v.as_str()).unwrap_or(&id).to_string();
return Ok((id, root));
}
Err(BenchError::Config(format!(
"No models found on the server at {base_url}"
)))
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
+123 -1
View File
@@ -1,14 +1,96 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::future::Future;
use std::time::Instant;
use indicatif::{ProgressBar, ProgressStyle};
use thiserror_ext::AsReport as _;
use crate::backends::{RequestFuncInput, RequestFuncOutput, get_backend};
use crate::cli::BackendKind;
use crate::error::{BenchError, Result};
/// Retry an operation until it succeeds or the readiness timeout expires.
pub(crate) async fn retry_with_timeout<T, F, Fut>(
timeout_seconds: u64,
retry_interval: u64,
mut operation: F,
) -> Result<T>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T>>,
{
if timeout_seconds == 0 {
return operation().await;
}
let deadline = Instant::now() + std::time::Duration::from_secs(timeout_seconds);
loop {
let error = match operation().await {
Ok(value) => return Ok(value),
Err(error) => error,
};
tracing::warn!(error = %error.as_report(), "endpoint is not ready");
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(BenchError::EndpointTimeout(
timeout_seconds,
format!("{}", error.as_report()),
));
}
let sleep_duration =
std::cmp::min(std::time::Duration::from_secs(retry_interval), remaining);
if !sleep_duration.is_zero() {
tokio::time::sleep(sleep_duration).await;
}
}
}
/// Fetch the first model from the server, retrying while it starts.
pub(crate) async fn get_first_model(
base_url: &str,
client: &reqwest::Client,
extra_headers: &Option<std::collections::HashMap<String, String>>,
timeout_seconds: u64,
) -> Result<(String, String)> {
let url = format!("{base_url}/v1/models");
retry_with_timeout(timeout_seconds, 5, || async {
let mut request = client.get(&url);
if let Some(headers) = extra_headers {
for (key, value) in headers {
request = request.header(key, value);
}
}
// Add API key from environment
if let Ok(api_key) = std::env::var("OPENAI_API_KEY") {
request = request.header("Authorization", format!("Bearer {api_key}"));
}
let response = request.send().await?.error_for_status()?;
let data: serde_json::Value = response.json().await?;
if let Some(model) = data
.get("data")
.and_then(|value| value.as_array())
.and_then(|models| models.first())
{
let id =
model.get("id").and_then(|value| value.as_str()).unwrap_or_default().to_string();
let root =
model.get("root").and_then(|value| value.as_str()).unwrap_or(&id).to_string();
return Ok((id, root));
}
Err(BenchError::Config(format!(
"No models found on the server at {base_url}"
)))
})
.await
}
/// Wait for the serving endpoint to become available.
///
/// Sends test requests with retry until success or timeout.
@@ -63,7 +145,7 @@ pub async fn wait_for_endpoint(
last_error = err;
}
Err(e) => {
last_error = e.to_string();
last_error = format!("{}", e.as_report());
}
}
@@ -76,3 +158,43 @@ pub async fn wait_for_endpoint(
Err(BenchError::EndpointTimeout(timeout_seconds, last_error))
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use super::*;
#[tokio::test]
async fn retry_succeeds_after_transient_failures() {
let attempts = AtomicUsize::new(0);
let value = retry_with_timeout(1, 0, || async {
if attempts.fetch_add(1, Ordering::Relaxed) < 2 {
Err(BenchError::Backend("not ready".into()))
} else {
Ok(42)
}
})
.await
.unwrap();
assert_eq!(value, 42);
assert_eq!(attempts.load(Ordering::Relaxed), 3);
}
#[tokio::test]
async fn zero_timeout_does_not_retry() {
let attempts = AtomicUsize::new(0);
let error = retry_with_timeout(0, 0, || async {
attempts.fetch_add(1, Ordering::Relaxed);
Err::<(), _>(BenchError::Backend("not ready".into()))
})
.await
.unwrap_err();
assert!(matches!(error, BenchError::Backend(_)));
assert_eq!(attempts.load(Ordering::Relaxed), 1);
}
}
+9 -6
View File
@@ -30,7 +30,7 @@ pub struct ServerTokenizer {
impl ServerTokenizer {
/// Create a new server tokenizer and verify connectivity.
pub async fn new(base_url: &str, model: &str) -> Result<Self> {
pub async fn new(base_url: &str, model: &str, timeout_seconds: u64) -> Result<Self> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()
@@ -49,7 +49,10 @@ impl ServerTokenizer {
};
// Probe the endpoint to verify it works and discover vocab size
let test_tokens = st.encode_async("test").await?;
let test_tokens = crate::ready_checker::retry_with_timeout(timeout_seconds, 5, || {
st.encode_async("test")
})
.await?;
let max_id = test_tokens.iter().copied().max().unwrap_or(0);
let estimated_vocab = (max_id * 2).max(131072);
@@ -225,11 +228,11 @@ impl TokenizerKind {
/// 2. Tiktoken model file (for Kimi, Qwen, etc.)
/// 3. Server-side /tokenize + /detokenize endpoints
///
/// `server_info` is `Some((base_url, model))` to enable server-side fallback.
/// `server_info` is `Some((base_url, model, timeout_seconds))` to enable server-side fallback.
pub async fn load_tokenizer(
model_id: &str,
_trust_remote_code: bool,
server_info: Option<(&str, &str)>,
server_info: Option<(&str, &str, u64)>,
) -> Result<TokenizerKind> {
// 0. Check for built-in tiktoken encoding names (no HF download needed). These are useful for
// consistent cross-model token counting (e.g. Artificial Analysis).
@@ -275,13 +278,13 @@ pub async fn load_tokenizer(
}
Err(tiktoken_err) => {
// 3. Try server-side fallback
if let Some((base_url, model)) = server_info {
if let Some((base_url, model, timeout_seconds)) = server_info {
tracing::info!(
model = model_id,
error = %tiktoken_err.as_report(),
"tiktoken unavailable; trying server-side tokenization"
);
match ServerTokenizer::new(base_url, model).await {
match ServerTokenizer::new(base_url, model, timeout_seconds).await {
Ok(srv) => {
tracing::info!(
model = model_id,