[Bugfix][Benchmark] Check readiness before tokenizer init in rust vllm-bench (#51863)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
03b87dc42f
commit
aabc1a0a0b
@@ -18,7 +18,7 @@ use crate::metrics::steady_state;
|
||||
use crate::output::console::print_results;
|
||||
use crate::output::json::{append_result, build_result_json, compute_result_filename, save_result};
|
||||
use crate::rate_control::compute_schedule;
|
||||
use crate::ready_checker::wait_for_endpoint;
|
||||
use crate::ready_checker::{get_first_model, wait_for_endpoint};
|
||||
|
||||
/// Pre-resolve the hostname in `base_url` and pin all resolved IPs on the
|
||||
/// client builder via [`reqwest::ClientBuilder::resolve_to_addrs`]. This
|
||||
@@ -281,40 +281,6 @@ pub(crate) fn assign_lora_modules(
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Fetch the first model from the server's /v1/models endpoint.
|
||||
async fn get_first_model_from_server(
|
||||
base_url: &str,
|
||||
client: &reqwest::Client,
|
||||
extra_headers: &Option<std::collections::HashMap<String, String>>,
|
||||
) -> Result<(String, String)> {
|
||||
let url = format!("{base_url}/v1/models");
|
||||
let mut request = client.get(&url);
|
||||
if let Some(headers) = extra_headers {
|
||||
for (k, v) in headers {
|
||||
request = request.header(k, v);
|
||||
}
|
||||
}
|
||||
// Add API key from environment
|
||||
if let Ok(api_key) = std::env::var("OPENAI_API_KEY") {
|
||||
request = request.header("Authorization", format!("Bearer {api_key}"));
|
||||
}
|
||||
|
||||
let response = request.send().await?;
|
||||
let data: serde_json::Value = response.json().await?;
|
||||
|
||||
if let Some(models) = data.get("data").and_then(|d| d.as_array())
|
||||
&& let Some(first) = models.first()
|
||||
{
|
||||
let id = first.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string();
|
||||
let root = first.get("root").and_then(|v| v.as_str()).unwrap_or(&id).to_string();
|
||||
return Ok((id, root));
|
||||
}
|
||||
|
||||
Err(BenchError::Config(format!(
|
||||
"No models found on the server at {base_url}"
|
||||
)))
|
||||
}
|
||||
|
||||
/// Run the complete benchmark.
|
||||
///
|
||||
/// This is the core orchestrator matching Python's benchmark() + main_async().
|
||||
@@ -348,8 +314,13 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
|
||||
(m.clone(), config.model_name.clone())
|
||||
} else {
|
||||
tracing::info!(base_url = %config.base_url, "fetching first model from server");
|
||||
let (name, id) =
|
||||
get_first_model_from_server(&config.base_url, &client, &config.extra_headers).await?;
|
||||
let (name, id) = get_first_model(
|
||||
&config.base_url,
|
||||
&client,
|
||||
&config.extra_headers,
|
||||
config.ready_check_timeout_sec,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
model_name = name,
|
||||
model_id = id,
|
||||
@@ -364,7 +335,11 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
|
||||
} else {
|
||||
let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id);
|
||||
tracing::info!(tokenizer = tid, "loading tokenizer");
|
||||
let server_info = Some((config.base_url.as_str(), model_id.as_str()));
|
||||
let server_info = Some((
|
||||
config.base_url.as_str(),
|
||||
model_id.as_str(),
|
||||
config.ready_check_timeout_sec,
|
||||
));
|
||||
let t =
|
||||
crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info).await?;
|
||||
Some(t)
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::output::console::print_multi_turn_results;
|
||||
use crate::output::json::{
|
||||
append_result, build_multi_turn_result_json, compute_result_filename, save_result,
|
||||
};
|
||||
use crate::ready_checker::{get_first_model, wait_for_endpoint};
|
||||
|
||||
/// Output from a single turn within a conversation.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -74,7 +75,13 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
|
||||
(m.clone(), config.model_name.clone())
|
||||
} else {
|
||||
tracing::info!(base_url = %config.base_url, "fetching first model from server");
|
||||
let (name, id) = get_first_model(&config.base_url, &client, &config.extra_headers).await?;
|
||||
let (name, id) = get_first_model(
|
||||
&config.base_url,
|
||||
&client,
|
||||
&config.extra_headers,
|
||||
config.ready_check_timeout_sec,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
model_name = name,
|
||||
model_id = id,
|
||||
@@ -89,7 +96,11 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
|
||||
} else {
|
||||
let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id);
|
||||
tracing::info!(tokenizer = tid, "loading tokenizer");
|
||||
let server_info = Some((config.base_url.as_str(), model_id.as_str()));
|
||||
let server_info = Some((
|
||||
config.base_url.as_str(),
|
||||
model_id.as_str(),
|
||||
config.ready_check_timeout_sec,
|
||||
));
|
||||
let t =
|
||||
crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info).await?;
|
||||
Some(t)
|
||||
@@ -263,7 +274,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
|
||||
};
|
||||
|
||||
tracing::info!("starting initial single-prompt test run");
|
||||
let test_output = crate::ready_checker::wait_for_endpoint(
|
||||
let test_output = wait_for_endpoint(
|
||||
config.backend,
|
||||
&client,
|
||||
&test_input,
|
||||
@@ -748,39 +759,6 @@ async fn run_conversation(
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the first model from the server's /v1/models endpoint.
|
||||
async fn get_first_model(
|
||||
base_url: &str,
|
||||
client: &reqwest::Client,
|
||||
extra_headers: &Option<HashMap<String, String>>,
|
||||
) -> Result<(String, String)> {
|
||||
let url = format!("{base_url}/v1/models");
|
||||
let mut request = client.get(&url);
|
||||
if let Some(headers) = extra_headers {
|
||||
for (k, v) in headers {
|
||||
request = request.header(k, v);
|
||||
}
|
||||
}
|
||||
if let Ok(api_key) = std::env::var("OPENAI_API_KEY") {
|
||||
request = request.header("Authorization", format!("Bearer {api_key}"));
|
||||
}
|
||||
|
||||
let response = request.send().await?;
|
||||
let data: serde_json::Value = response.json().await?;
|
||||
|
||||
if let Some(models) = data.get("data").and_then(|d| d.as_array())
|
||||
&& let Some(first) = models.first()
|
||||
{
|
||||
let id = first.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string();
|
||||
let root = first.get("root").and_then(|v| v.as_str()).unwrap_or(&id).to_string();
|
||||
return Ok((id, root));
|
||||
}
|
||||
|
||||
Err(BenchError::Config(format!(
|
||||
"No models found on the server at {base_url}"
|
||||
)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -1,14 +1,96 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::future::Future;
|
||||
use std::time::Instant;
|
||||
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use crate::backends::{RequestFuncInput, RequestFuncOutput, get_backend};
|
||||
use crate::cli::BackendKind;
|
||||
use crate::error::{BenchError, Result};
|
||||
|
||||
/// Retry an operation until it succeeds or the readiness timeout expires.
|
||||
pub(crate) async fn retry_with_timeout<T, F, Fut>(
|
||||
timeout_seconds: u64,
|
||||
retry_interval: u64,
|
||||
mut operation: F,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T>>,
|
||||
{
|
||||
if timeout_seconds == 0 {
|
||||
return operation().await;
|
||||
}
|
||||
|
||||
let deadline = Instant::now() + std::time::Duration::from_secs(timeout_seconds);
|
||||
|
||||
loop {
|
||||
let error = match operation().await {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(error) => error,
|
||||
};
|
||||
tracing::warn!(error = %error.as_report(), "endpoint is not ready");
|
||||
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(BenchError::EndpointTimeout(
|
||||
timeout_seconds,
|
||||
format!("{}", error.as_report()),
|
||||
));
|
||||
}
|
||||
|
||||
let sleep_duration =
|
||||
std::cmp::min(std::time::Duration::from_secs(retry_interval), remaining);
|
||||
if !sleep_duration.is_zero() {
|
||||
tokio::time::sleep(sleep_duration).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the first model from the server, retrying while it starts.
|
||||
pub(crate) async fn get_first_model(
|
||||
base_url: &str,
|
||||
client: &reqwest::Client,
|
||||
extra_headers: &Option<std::collections::HashMap<String, String>>,
|
||||
timeout_seconds: u64,
|
||||
) -> Result<(String, String)> {
|
||||
let url = format!("{base_url}/v1/models");
|
||||
retry_with_timeout(timeout_seconds, 5, || async {
|
||||
let mut request = client.get(&url);
|
||||
if let Some(headers) = extra_headers {
|
||||
for (key, value) in headers {
|
||||
request = request.header(key, value);
|
||||
}
|
||||
}
|
||||
// Add API key from environment
|
||||
if let Ok(api_key) = std::env::var("OPENAI_API_KEY") {
|
||||
request = request.header("Authorization", format!("Bearer {api_key}"));
|
||||
}
|
||||
|
||||
let response = request.send().await?.error_for_status()?;
|
||||
let data: serde_json::Value = response.json().await?;
|
||||
if let Some(model) = data
|
||||
.get("data")
|
||||
.and_then(|value| value.as_array())
|
||||
.and_then(|models| models.first())
|
||||
{
|
||||
let id =
|
||||
model.get("id").and_then(|value| value.as_str()).unwrap_or_default().to_string();
|
||||
let root =
|
||||
model.get("root").and_then(|value| value.as_str()).unwrap_or(&id).to_string();
|
||||
return Ok((id, root));
|
||||
}
|
||||
|
||||
Err(BenchError::Config(format!(
|
||||
"No models found on the server at {base_url}"
|
||||
)))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Wait for the serving endpoint to become available.
|
||||
///
|
||||
/// Sends test requests with retry until success or timeout.
|
||||
@@ -63,7 +145,7 @@ pub async fn wait_for_endpoint(
|
||||
last_error = err;
|
||||
}
|
||||
Err(e) => {
|
||||
last_error = e.to_string();
|
||||
last_error = format!("{}", e.as_report());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,3 +158,43 @@ pub async fn wait_for_endpoint(
|
||||
|
||||
Err(BenchError::EndpointTimeout(timeout_seconds, last_error))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_succeeds_after_transient_failures() {
|
||||
let attempts = AtomicUsize::new(0);
|
||||
|
||||
let value = retry_with_timeout(1, 0, || async {
|
||||
if attempts.fetch_add(1, Ordering::Relaxed) < 2 {
|
||||
Err(BenchError::Backend("not ready".into()))
|
||||
} else {
|
||||
Ok(42)
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(value, 42);
|
||||
assert_eq!(attempts.load(Ordering::Relaxed), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_timeout_does_not_retry() {
|
||||
let attempts = AtomicUsize::new(0);
|
||||
|
||||
let error = retry_with_timeout(0, 0, || async {
|
||||
attempts.fetch_add(1, Ordering::Relaxed);
|
||||
Err::<(), _>(BenchError::Backend("not ready".into()))
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(error, BenchError::Backend(_)));
|
||||
assert_eq!(attempts.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ pub struct ServerTokenizer {
|
||||
|
||||
impl ServerTokenizer {
|
||||
/// Create a new server tokenizer and verify connectivity.
|
||||
pub async fn new(base_url: &str, model: &str) -> Result<Self> {
|
||||
pub async fn new(base_url: &str, model: &str, timeout_seconds: u64) -> Result<Self> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
@@ -49,7 +49,10 @@ impl ServerTokenizer {
|
||||
};
|
||||
|
||||
// Probe the endpoint to verify it works and discover vocab size
|
||||
let test_tokens = st.encode_async("test").await?;
|
||||
let test_tokens = crate::ready_checker::retry_with_timeout(timeout_seconds, 5, || {
|
||||
st.encode_async("test")
|
||||
})
|
||||
.await?;
|
||||
let max_id = test_tokens.iter().copied().max().unwrap_or(0);
|
||||
let estimated_vocab = (max_id * 2).max(131072);
|
||||
|
||||
@@ -225,11 +228,11 @@ impl TokenizerKind {
|
||||
/// 2. Tiktoken model file (for Kimi, Qwen, etc.)
|
||||
/// 3. Server-side /tokenize + /detokenize endpoints
|
||||
///
|
||||
/// `server_info` is `Some((base_url, model))` to enable server-side fallback.
|
||||
/// `server_info` is `Some((base_url, model, timeout_seconds))` to enable server-side fallback.
|
||||
pub async fn load_tokenizer(
|
||||
model_id: &str,
|
||||
_trust_remote_code: bool,
|
||||
server_info: Option<(&str, &str)>,
|
||||
server_info: Option<(&str, &str, u64)>,
|
||||
) -> Result<TokenizerKind> {
|
||||
// 0. Check for built-in tiktoken encoding names (no HF download needed). These are useful for
|
||||
// consistent cross-model token counting (e.g. Artificial Analysis).
|
||||
@@ -275,13 +278,13 @@ pub async fn load_tokenizer(
|
||||
}
|
||||
Err(tiktoken_err) => {
|
||||
// 3. Try server-side fallback
|
||||
if let Some((base_url, model)) = server_info {
|
||||
if let Some((base_url, model, timeout_seconds)) = server_info {
|
||||
tracing::info!(
|
||||
model = model_id,
|
||||
error = %tiktoken_err.as_report(),
|
||||
"tiktoken unavailable; trying server-side tokenization"
|
||||
);
|
||||
match ServerTokenizer::new(base_url, model).await {
|
||||
match ServerTokenizer::new(base_url, model, timeout_seconds).await {
|
||||
Ok(srv) => {
|
||||
tracing::info!(
|
||||
model = model_id,
|
||||
|
||||
Reference in New Issue
Block a user