[Rust][Benchmark] Use tracing for logs (#48937)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-21 05:18:13 +00:00
committed by GitHub
parent f25953cc59
commit 8688a06d67
24 changed files with 498 additions and 249 deletions
+3
View File
@@ -5498,10 +5498,13 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"thiserror 2.0.18", "thiserror 2.0.18",
"thiserror-ext",
"tiktoken-rs 0.9.1", "tiktoken-rs 0.9.1",
"tokenizers", "tokenizers",
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"tracing",
"tracing-subscriber",
"url", "url",
"uuid", "uuid",
] ]
+3
View File
@@ -26,10 +26,13 @@ rustc-hash.workspace = true
serde = { workspace = true, features = ["rc"] } serde = { workspace = true, features = ["rc"] }
serde_json = { workspace = true, features = ["raw_value"] } serde_json = { workspace = true, features = ["raw_value"] }
thiserror.workspace = true thiserror.workspace = true
thiserror-ext.workspace = true
tiktoken-rs.workspace = true tiktoken-rs.workspace = true
tokenizers.workspace = true tokenizers.workspace = true
tokio.workspace = true tokio.workspace = true
tokio-stream.workspace = true tokio-stream.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
url.workspace = true url.workspace = true
uuid.workspace = true uuid.workspace = true
+8 -7
View File
@@ -158,9 +158,10 @@ impl PoolingBackend {
// (mirrors Python async_request_vllm_rerank). // (mirrors Python async_request_vllm_rerank).
if let Some(ref list) = input.prompt_list { if let Some(ref list) = input.prompt_list {
if list.len() < 2 { if list.len() < 2 {
eprintln!( tracing::warn!(
"WARNING: vllm-rerank request has no documents \ backend = "vllm-rerank",
(prompt_list needs [query, doc, ...])" inputs = list.len(),
"rerank request has no documents"
); );
} }
let query = list.first().map(|s| s.as_ref()).unwrap_or(""); let query = list.first().map(|s| s.as_ref()).unwrap_or("");
@@ -175,10 +176,10 @@ impl PoolingBackend {
// Legacy path: text prompt as query, documents via --extra-body. // Legacy path: text prompt as query, documents via --extra-body.
let query = input.prompt.as_ref(); let query = input.prompt.as_ref();
if query.is_empty() && input.prompt_token_ids.is_some() { if query.is_empty() && input.prompt_token_ids.is_some() {
eprintln!( tracing::warn!(
"WARNING: vllm-rerank received empty query (random dataset uses \ backend = "vllm-rerank",
token IDs only). Use --dataset-name random-rerank for meaningful \ dataset = "random",
rerank benchmarks." "rerank request has an empty query; use the random-rerank dataset"
); );
} }
serde_json::json!({ serde_json::json!({
+107 -80
View File
@@ -6,6 +6,7 @@ use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use indicatif::{ProgressBar, ProgressStyle}; use indicatif::{ProgressBar, ProgressStyle};
use thiserror_ext::AsReport as _;
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
use crate::backends::{RequestFuncInput, RequestFuncOutput, get_backend}; use crate::backends::{RequestFuncInput, RequestFuncOutput, get_backend};
@@ -72,12 +73,12 @@ pub fn pre_resolve_dns(
v4.extend(v6); v4.extend(v6);
if !v4.is_empty() { if !v4.is_empty() {
let ips: Vec<_> = v4.iter().map(|a| a.ip()).collect(); let ips: Vec<_> = v4.iter().map(|a| a.ip()).collect();
println!("Pre-resolved {host} -> {ips:?}"); tracing::info!(host, addresses = ?ips, "pre-resolved benchmark endpoint DNS");
builder = builder.resolve_to_addrs(host, &v4); builder = builder.resolve_to_addrs(host, &v4);
} }
} }
Err(e) => { Err(e) => {
eprintln!("Warning: DNS pre-resolution for '{host}' failed: {e}"); tracing::warn!(host, error = %e.as_report(), "failed to pre-resolve benchmark endpoint DNS");
} }
} }
@@ -346,10 +347,14 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
let (model_id, model_name) = if let Some(ref m) = config.model { let (model_id, model_name) = if let Some(ref m) = config.model {
(m.clone(), config.model_name.clone()) (m.clone(), config.model_name.clone())
} else { } else {
println!("Model not specified, fetching first model from server..."); tracing::info!(base_url = %config.base_url, "fetching first model from server");
let (name, id) = let (name, id) =
get_first_model_from_server(&config.base_url, &client, &config.extra_headers).await?; get_first_model_from_server(&config.base_url, &client, &config.extra_headers).await?;
println!("First model name: {name}, first model id: {id}"); tracing::info!(
model_name = name,
model_id = id,
"selected first model from server"
);
(id, Some(name)) (id, Some(name))
}; };
@@ -358,10 +363,9 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
None None
} else { } else {
let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id); let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id);
println!("Loading tokenizer: {tid}"); tracing::info!(tokenizer = tid, "loading tokenizer");
let server_info = Some((config.base_url.as_str(), model_id.as_str())); let server_info = Some((config.base_url.as_str(), model_id.as_str()));
let t = crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info)?; let t = crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info)?;
println!("Tokenizer loaded successfully.");
Some(t) Some(t)
}; };
let has_tokenizer = tokenizer.is_some(); let has_tokenizer = tokenizer.is_some();
@@ -421,7 +425,12 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
config.num_prompts, config.random_batch_size, config.is_reranker, config.num_prompts, config.random_batch_size, config.is_reranker,
), ),
}; };
println!("Generating {dataset_label}..."); tracing::info!(
dataset = ?config.dataset_name,
prompts = config.num_prompts,
description = %dataset_label,
"generating benchmark dataset"
);
let gen_start = Instant::now(); let gen_start = Instant::now();
let mut input_requests = match config.dataset_name { let mut input_requests = match config.dataset_name {
@@ -608,18 +617,19 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
}; };
let gen_elapsed = gen_start.elapsed(); let gen_elapsed = gen_start.elapsed();
println!( tracing::info!(
"Generated {} prompts in {:.2}s", prompts = input_requests.len(),
input_requests.len(), elapsed_seconds = gen_elapsed.as_secs_f64(),
gen_elapsed.as_secs_f64() "generated benchmark dataset"
); );
let filtered_count = let filtered_count =
filter_requests_by_max_model_len(&mut input_requests, config.max_model_len); filter_requests_by_max_model_len(&mut input_requests, config.max_model_len);
if filtered_count > 0 { if filtered_count > 0 {
println!( tracing::info!(
"Filtered {filtered_count} prompt(s) above --max-model-len {}.", filtered_prompts = filtered_count,
config.max_model_len.unwrap() max_model_len = config.max_model_len.unwrap(),
"filtered prompts above maximum model length"
); );
} }
if input_requests.is_empty() { if input_requests.is_empty() {
@@ -670,7 +680,7 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
// Ready check // Ready check
if config.ready_check_timeout_sec > 0 { if config.ready_check_timeout_sec > 0 {
println!("Starting initial single prompt test run..."); tracing::info!("starting initial single-prompt test run");
let test_output = wait_for_endpoint( let test_output = wait_for_endpoint(
config.backend, config.backend,
&client, &client,
@@ -685,7 +695,7 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
test_output.error test_output.error
))); )));
} }
println!("Initial test run completed."); tracing::info!("initial single-prompt test run completed");
} }
// Verify and fix prompt token lengths against the server's /tokenize endpoint. // Verify and fix prompt token lengths against the server's /tokenize endpoint.
@@ -703,12 +713,15 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
DatasetName::Random | DatasetName::PrefixRepetition DatasetName::Random | DatasetName::PrefixRepetition
); );
if verifiable_dataset && has_token_ids && !config.backend.is_pooling() { if verifiable_dataset && has_token_ids && !config.backend.is_pooling() {
println!("Using prompt_token_ids, skipping server-side tokenizer verification."); tracing::info!(
reason = "prompt_token_ids",
"skipping server tokenizer verification"
);
} }
if verifiable_dataset && !has_token_ids && !config.backend.is_pooling() { if verifiable_dataset && !has_token_ids && !config.backend.is_pooling() {
let cache_key = tokenizer_verify_cache_key(&config.base_url, &model_id); let cache_key = tokenizer_verify_cache_key(&config.base_url, &model_id);
if is_tokenizer_verified(&cache_key) { if is_tokenizer_verified(&cache_key) {
println!("Tokenizer verified in previous run (cached), skipping verification."); tracing::info!(reason = "cached", "skipping server tokenizer verification");
} else { } else {
let num_special = let num_special =
tokenizer.as_ref().map(|t| t.num_special_tokens_to_add()).unwrap_or(0); tokenizer.as_ref().map(|t| t.num_special_tokens_to_add()).unwrap_or(0);
@@ -723,14 +736,17 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
.await? .await?
{ {
SampleVerifyOutcome::Passed => { SampleVerifyOutcome::Passed => {
println!("Sample verification passed, skipping full verification."); tracing::info!("tokenizer sample verification passed");
mark_tokenizer_verified(&cache_key); mark_tokenizer_verified(&cache_key);
} }
SampleVerifyOutcome::Skipped(reason) => { SampleVerifyOutcome::Skipped(reason) => {
println!("Server /tokenize unavailable ({reason}), skipping verification."); tracing::warn!(
reason = %reason,
"server tokenizer unavailable; skipping prompt verification"
);
} }
SampleVerifyOutcome::Mismatch => { SampleVerifyOutcome::Mismatch => {
println!("Sample verification found mismatch, running full verify+fix..."); tracing::warn!("tokenizer sample mismatch; verifying and fixing all prompts");
match verify_and_fix_prompt_lengths( match verify_and_fix_prompt_lengths(
&client, &client,
&config.base_url, &config.base_url,
@@ -742,16 +758,16 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
.await .await
{ {
Ok(()) => { Ok(()) => {
println!( tracing::info!(
"All {} prompts verified: exact token length match.", prompts = input_requests.len(),
input_requests.len() "verified exact prompt token lengths"
); );
mark_tokenizer_verified(&cache_key); mark_tokenizer_verified(&cache_key);
} }
Err(BenchError::TokenizeUnavailable(reason)) => { Err(BenchError::TokenizeUnavailable(reason)) => {
println!( tracing::warn!(
"Server /tokenize became unavailable during verification \ reason = %reason,
({reason}); proceeding with client-side token counts." "server tokenizer became unavailable; using client token counts"
); );
} }
Err(e) => return Err(e), Err(e) => return Err(e),
@@ -763,7 +779,7 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
// Warmup // Warmup
if config.num_warmups > 0 { if config.num_warmups > 0 {
println!("Warming up with {} requests...", config.num_warmups); tracing::info!(requests = config.num_warmups, "starting benchmark warmup");
run_warmup( run_warmup(
config.backend, config.backend,
&client, &client,
@@ -776,7 +792,7 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
config.disable_tqdm, config.disable_tqdm,
) )
.await; .await;
println!("Warmup run completed."); tracing::info!(requests = config.num_warmups, "benchmark warmup completed");
} }
// Start profiler if requested (immediate mode — no batch threshold) // Start profiler if requested (immediate mode — no batch threshold)
@@ -814,28 +830,22 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
let spec_decode_before = let spec_decode_before =
fetch_spec_decode_metrics(&config.base_url, &client, &config.extra_headers).await; fetch_spec_decode_metrics(&config.base_url, &client, &config.extra_headers).await;
if spec_decode_before.is_some() { if spec_decode_before.is_some() {
println!("Speculative decoding detected, will collect metrics."); tracing::info!("detected speculative decoding; collecting metrics");
} }
// Main benchmark // Main benchmark
println!("Starting main benchmark run...");
let distribution = if config.burstiness == 1.0 { let distribution = if config.burstiness == 1.0 {
"Poisson process" "Poisson process"
} else { } else {
"Gamma distribution" "Gamma distribution"
}; };
println!( tracing::info!(
"Traffic request rate: {}", request_rate = config.request_rate,
if config.request_rate.is_infinite() { burstiness = config.burstiness,
"inf".to_string() distribution,
} else { max_concurrency = config.max_concurrency.unwrap_or(config.num_prompts),
format!("{}", config.request_rate) prompts = config.num_prompts,
} "starting main benchmark run"
);
println!("Burstiness factor: {} ({distribution})", config.burstiness);
println!(
"Maximum request concurrency: {}",
config.max_concurrency.unwrap_or(config.num_prompts)
); );
// Pre-assign LoRA adapters to each request (None when --lora-modules not set). // Pre-assign LoRA adapters to each request (None when --lora-modules not set).
@@ -847,11 +857,11 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
); );
if let (Some(modules), Some(_)) = (config.lora_modules.as_ref(), lora_assignments.as_ref()) { if let (Some(modules), Some(_)) = (config.lora_modules.as_ref(), lora_assignments.as_ref()) {
let names: Vec<&str> = modules.iter().map(|s| s.as_ref()).collect(); let names: Vec<&str> = modules.iter().map(|s| s.as_ref()).collect();
println!( tracing::info!(
"LoRA adapters ({}): {:?} [assignment={:?}]", adapters = modules.len(),
modules.len(), names = ?names,
names, assignment = ?config.lora_assignment,
config.lora_assignment "assigned LoRA adapters"
); );
} }
@@ -1125,7 +1135,7 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
if let Some((cancel_tx, task)) = profile_task { if let Some((cancel_tx, task)) = profile_task {
let _ = cancel_tx.send(()); let _ = cancel_tx.send(());
if let Err(e) = task.await { if let Err(e) = task.await {
eprintln!("WARNING: Profile background task failed: {e}"); tracing::error!(error = %e.as_report(), "profiler background task failed");
} }
} }
@@ -1289,12 +1299,14 @@ pub(crate) async fn start_profiler_immediate(
base_url: &str, base_url: &str,
extra_headers: &Option<std::collections::HashMap<String, String>>, extra_headers: &Option<std::collections::HashMap<String, String>>,
) { ) {
println!("Starting profiler...");
let profile_url = format!("{base_url}/start_profile"); let profile_url = format!("{base_url}/start_profile");
tracing::info!(url = %profile_url, "starting profiler");
match send_profile_request(client, &profile_url, extra_headers).await { match send_profile_request(client, &profile_url, extra_headers).await {
Ok(true) => println!("Profiler started"), Ok(true) => tracing::info!(url = %profile_url, "profiler started"),
Ok(false) => eprintln!("WARNING: Profiler start request returned non-success"), Ok(false) => tracing::warn!(url = %profile_url, "profiler start request was unsuccessful"),
Err(e) => eprintln!("WARNING: Failed to start profiler: {e}"), Err(e) => {
tracing::warn!(url = %profile_url, error = %e.as_report(), "failed to start profiler")
}
} }
} }
@@ -1304,12 +1316,14 @@ pub(crate) async fn stop_profiler_immediate(
base_url: &str, base_url: &str,
extra_headers: &Option<std::collections::HashMap<String, String>>, extra_headers: &Option<std::collections::HashMap<String, String>>,
) { ) {
println!("Stopping profiler...");
let profile_url = format!("{base_url}/stop_profile"); let profile_url = format!("{base_url}/stop_profile");
tracing::info!(url = %profile_url, "stopping profiler");
match send_profile_request(client, &profile_url, extra_headers).await { match send_profile_request(client, &profile_url, extra_headers).await {
Ok(true) => println!("Profiler stopped"), Ok(true) => tracing::info!(url = %profile_url, "profiler stopped"),
Ok(false) => eprintln!("WARNING: Profiler stop request returned non-success"), Ok(false) => tracing::warn!(url = %profile_url, "profiler stop request was unsuccessful"),
Err(e) => eprintln!("WARNING: Failed to stop profiler: {e}"), Err(e) => {
tracing::warn!(url = %profile_url, error = %e.as_report(), "failed to stop profiler")
}
} }
} }
@@ -1371,25 +1385,30 @@ pub(crate) async fn profile_on_batch_threshold(
duration_secs: f64, duration_secs: f64,
mut cancel_rx: tokio::sync::oneshot::Receiver<()>, mut cancel_rx: tokio::sync::oneshot::Receiver<()>,
) { ) {
println!( tracing::info!(
"Waiting for batch size >= {threshold} before starting profiler \ threshold,
(will capture {duration_secs}s)..." duration_seconds = duration_secs,
"waiting for profiler batch threshold"
); );
loop { loop {
if let Some(running) = fetch_num_requests_running(client, base_url).await if let Some(running) = fetch_num_requests_running(client, base_url).await
&& running >= threshold && running >= threshold
{ {
println!("Batch size {running} >= {threshold}, starting profiler..."); tracing::info!(
running_requests = running,
threshold,
"profiler batch threshold reached"
);
break; break;
} }
// Wait 500ms or until the benchmark signals cancellation // Wait 500ms or until the benchmark signals cancellation
tokio::select! { tokio::select! {
_ = tokio::time::sleep(std::time::Duration::from_millis(500)) => {} _ = tokio::time::sleep(std::time::Duration::from_millis(500)) => {}
_ = &mut cancel_rx => { _ = &mut cancel_rx => {
eprintln!( tracing::warn!(
"NOTE: Benchmark finished before batch threshold {threshold} was reached; \ threshold,
profiling skipped." "benchmark finished before profiler batch threshold; skipping profiling"
); );
return; return;
} }
@@ -1398,13 +1417,13 @@ pub(crate) async fn profile_on_batch_threshold(
let start_url = format!("{base_url}/start_profile"); let start_url = format!("{base_url}/start_profile");
match send_profile_request(client, &start_url, extra_headers).await { match send_profile_request(client, &start_url, extra_headers).await {
Ok(true) => println!("Profiler started"), Ok(true) => tracing::info!(url = %start_url, "profiler started"),
Ok(false) => { Ok(false) => {
eprintln!("WARNING: Profiler start request returned non-success"); tracing::warn!(url = %start_url, "profiler start request was unsuccessful");
return; return;
} }
Err(e) => { Err(e) => {
eprintln!("WARNING: Failed to start profiler: {e}"); tracing::warn!(url = %start_url, error = %e.as_report(), "failed to start profiler");
return; return;
} }
} }
@@ -1413,15 +1432,17 @@ pub(crate) async fn profile_on_batch_threshold(
tokio::select! { tokio::select! {
_ = tokio::time::sleep(std::time::Duration::from_secs_f64(duration_secs)) => {} _ = tokio::time::sleep(std::time::Duration::from_secs_f64(duration_secs)) => {}
_ = &mut cancel_rx => { _ = &mut cancel_rx => {
println!("Benchmark finished, stopping profiler early..."); tracing::info!("benchmark finished; stopping profiler early");
} }
} }
let stop_url = format!("{base_url}/stop_profile"); let stop_url = format!("{base_url}/stop_profile");
match send_profile_request(client, &stop_url, extra_headers).await { match send_profile_request(client, &stop_url, extra_headers).await {
Ok(true) => println!("Profiler stopped after capturing"), Ok(true) => tracing::info!(url = %stop_url, "profiler stopped after capture"),
Ok(false) => eprintln!("WARNING: Profiler stop request returned non-success"), Ok(false) => tracing::warn!(url = %stop_url, "profiler stop request was unsuccessful"),
Err(e) => eprintln!("WARNING: Failed to stop profiler: {e}"), Err(e) => {
tracing::warn!(url = %stop_url, error = %e.as_report(), "failed to stop profiler")
}
} }
} }
@@ -1502,10 +1523,11 @@ async fn verify_and_fix_prompt_lengths(
let excess = tokens.len().saturating_sub(expected_input_len); let excess = tokens.len().saturating_sub(expected_input_len);
let compensate = if excess > 0 && last_excess == Some(excess) { let compensate = if excess > 0 && last_excess == Some(excess) {
if _iter == 1 { if _iter == 1 {
eprintln!( tracing::warn!(
"Prompt {i}: server consistently adds {excess} extra token(s) \ prompt_index = i,
(likely BOS), compensating target to {}.", extra_tokens = excess,
expected_input_len.saturating_sub(excess), adjusted_target = expected_input_len.saturating_sub(excess),
"server consistently adds prompt tokens; compensating verification target"
); );
} }
excess excess
@@ -1563,7 +1585,10 @@ async fn verify_and_fix_prompt_lengths(
let fc = fixed_count.load(std::sync::atomic::Ordering::Relaxed); let fc = fixed_count.load(std::sync::atomic::Ordering::Relaxed);
if fc > 0 { if fc > 0 {
println!("Fixed {fc} prompt(s) via server tokenize/detokenize convergence."); tracing::info!(
fixed_prompts = fc,
"fixed prompt lengths using server tokenizer"
);
} }
Ok(()) Ok(())
@@ -1818,7 +1843,7 @@ async fn sample_verify_prompts(
let tokenize_url = format!("{base_url}/tokenize"); let tokenize_url = format!("{base_url}/tokenize");
let api_key = std::env::var("OPENAI_API_KEY").ok(); let api_key = std::env::var("OPENAI_API_KEY").ok();
println!("Sampling {sample_size} prompts for verification..."); tracing::info!(sample_size, "sampling prompts for tokenizer verification");
for (i, request) in requests.iter().enumerate().take(sample_size) { for (i, request) in requests.iter().enumerate().take(sample_size) {
let tokens = match server_tokenize( let tokens = match server_tokenize(
@@ -1841,9 +1866,11 @@ async fn sample_verify_prompts(
let expected = request.prompt_len + num_special; let expected = request.prompt_len + num_special;
if tokens.len() != expected { if tokens.len() != expected {
println!( tracing::warn!(
"Prompt {i}: expected {expected} tokens, server returned {}", prompt_index = i,
tokens.len() expected_tokens = expected,
actual_tokens = tokens.len(),
"tokenizer verification sample mismatch"
); );
return Ok(SampleVerifyOutcome::Mismatch); return Ok(SampleVerifyOutcome::Mismatch);
} }
+18 -9
View File
@@ -288,10 +288,18 @@ impl BenchConfig {
} }
Some(other) => { Some(other) => {
// extra_body was not an object — just use sampling params // extra_body was not an object — just use sampling params
eprintln!( let value_type = match &other {
"Warning: --extra-body is not a JSON object, sampling params may be lost" serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "boolean",
serde_json::Value::Number(_) => "number",
serde_json::Value::String(_) => "string",
serde_json::Value::Array(_) => "array",
serde_json::Value::Object(_) => unreachable!(),
};
tracing::warn!(
value_type,
"sampling parameters may be lost because --extra-body is not a JSON object"
); );
let _ = other;
sampling_params sampling_params
} }
None => sampling_params, None => sampling_params,
@@ -489,9 +497,9 @@ impl BenchConfig {
_ => {} _ => {}
} }
if !args.skip_chat_template { if !args.skip_chat_template {
eprintln!( tracing::warn!(
"NOTE: client-side chat template rendering is not supported; custom \ dataset = "custom",
dataset prompts are sent raw (equivalent to --skip-chat-template)." "client-side chat template rendering is unsupported; sending prompts raw"
); );
} }
} }
@@ -570,9 +578,10 @@ impl BenchConfig {
} }
if ignore_eos { if ignore_eos {
eprintln!( tracing::warn!(
"WARNING: --ignore-eos is set with --multi-turn. The server may not \ ignore_eos,
respect output length limits, causing unbounded context growth." multi_turn = true,
"output length limits may be ignored, causing unbounded context growth"
); );
} }
+59 -26
View File
@@ -8,6 +8,7 @@ use rand::seq::SliceRandom;
use rand::{Rng, SeedableRng}; use rand::{Rng, SeedableRng};
use super::SampleRequest; use super::SampleRequest;
use super::progress::RowDownloadReporter;
use crate::error::{BenchError, Result}; use crate::error::{BenchError, Result};
use crate::tokenizer::TokenizerKind; use crate::tokenizer::TokenizerKind;
@@ -201,7 +202,12 @@ pub fn download_hf_dataset(
(resolved_config, resolved_split) (resolved_config, resolved_split)
}; };
println!("HF dataset: {dataset} (config={resolved_config}, split={resolved_split})"); tracing::info!(
dataset,
config = resolved_config,
split = resolved_split,
"resolved Hugging Face dataset"
);
// Check cache // Check cache
let dir = cache_dir(); let dir = cache_dir();
@@ -215,11 +221,16 @@ pub fn download_hf_dataset(
if cache_path.exists() { if cache_path.exists() {
let path_str = cache_path.to_string_lossy().to_string(); let path_str = cache_path.to_string_lossy().to_string();
println!("HF dataset cached: {path_str}"); tracing::info!(dataset, path = %path_str, "using cached Hugging Face dataset");
return Ok((path_str, resolved_config, resolved_split)); return Ok((path_str, resolved_config, resolved_split));
} }
println!("Downloading HF dataset '{dataset}' from datasets-server..."); tracing::info!(
dataset,
config = resolved_config,
split = resolved_split,
"downloading Hugging Face dataset"
);
let encoded_config: String = let encoded_config: String =
url::form_urlencoded::byte_serialize(resolved_config.as_bytes()).collect(); url::form_urlencoded::byte_serialize(resolved_config.as_bytes()).collect();
@@ -229,6 +240,7 @@ pub fn download_hf_dataset(
let mut all_rows: Vec<serde_json::Value> = Vec::new(); let mut all_rows: Vec<serde_json::Value> = Vec::new();
let mut offset = 0usize; let mut offset = 0usize;
let page_size = 100usize; let page_size = 100usize;
let mut progress = RowDownloadReporter::new();
loop { loop {
let url = format!( let url = format!(
@@ -260,14 +272,14 @@ pub fn download_hf_dataset(
offset += fetched; offset += fetched;
let total = data["num_rows_total"].as_u64().unwrap_or(0); let total = data["num_rows_total"].as_u64().unwrap_or(0);
eprint!("\r Fetched {offset}/{total} rows..."); progress.update(offset, total);
// Stop if we have enough rows or reached end of dataset // Stop if we have enough rows or reached end of dataset
if all_rows.len() >= num_rows_needed || fetched < page_size { if all_rows.len() >= num_rows_needed || fetched < page_size {
break; break;
} }
} }
eprintln!(); // newline after progress progress.finish();
if all_rows.is_empty() { if all_rows.is_empty() {
return Err(BenchError::Config(format!( return Err(BenchError::Config(format!(
@@ -280,7 +292,12 @@ pub fn download_hf_dataset(
std::fs::write(&cache_path, &json_str)?; std::fs::write(&cache_path, &json_str)?;
let path_str = cache_path.to_string_lossy().to_string(); let path_str = cache_path.to_string_lossy().to_string();
println!("HF dataset: {} rows saved to {path_str}", all_rows.len()); tracing::info!(
dataset,
rows = all_rows.len(),
path = %path_str,
"saved Hugging Face dataset"
);
Ok((path_str, resolved_config, resolved_split)) Ok((path_str, resolved_config, resolved_split))
} }
@@ -483,21 +500,31 @@ pub fn load_hf_dataset(
// Detect column format from first row // Detect column format from first row
let format = detect_column_format(&entries[0], text_column_override)?; let format = detect_column_format(&entries[0], text_column_override)?;
// Print detected format
match &format { match &format {
ColumnFormat::Chat(col) => println!("HF dataset: detected chat column '{col}'"), ColumnFormat::Chat(col) => {
tracing::info!(
format = "chat",
column = col,
"detected Hugging Face dataset format"
);
}
ColumnFormat::Text { ColumnFormat::Text {
prompt_col, prompt_col,
output_col, output_col,
} => { } => {
let out_msg = output_col.as_deref().unwrap_or("none"); tracing::info!(
println!("HF dataset: detected text column '{prompt_col}', output column: {out_msg}"); format = "text",
prompt_column = prompt_col,
output_column = output_col.as_deref().unwrap_or("none"),
"detected Hugging Face dataset format"
);
} }
ColumnFormat::Combined { cols, output_col } => { ColumnFormat::Combined { cols, output_col } => {
let out_msg = output_col.as_deref().unwrap_or("none"); tracing::info!(
println!( format = "combined",
"HF dataset: detected combined columns {:?}, output column: {out_msg}", prompt_columns = ?cols,
cols output_column = output_col.as_deref().unwrap_or("none"),
"detected Hugging Face dataset format"
); );
} }
} }
@@ -608,9 +635,10 @@ pub fn load_hf_dataset(
if len == 0 { 128 } else { len } if len == 0 { 128 } else { len }
} else { } else {
if !warned_no_output { if !warned_no_output {
eprintln!( tracing::warn!(
"WARNING: No output column detected and --hf-output-len not set. \ path = dataset_path,
Using default output length of 128 tokens." default_output_tokens = 128,
"no dataset output column or --hf-output-len; using default output length"
); );
warned_no_output = true; warned_no_output = true;
} }
@@ -618,9 +646,10 @@ pub fn load_hf_dataset(
} }
} else { } else {
if !warned_no_output { if !warned_no_output {
eprintln!( tracing::warn!(
"WARNING: No output column detected and --hf-output-len not set. \ path = dataset_path,
Using default output length of 128 tokens." default_output_tokens = 128,
"no dataset output column or --hf-output-len; using default output length"
); );
warned_no_output = true; warned_no_output = true;
} }
@@ -640,9 +669,11 @@ pub fn load_hf_dataset(
// Oversample if needed // Oversample if needed
if samples.len() < num_requests { if samples.len() < num_requests {
if no_oversample { if no_oversample {
println!( tracing::info!(
"Skipping oversampling. Total samples: {} (requested: {num_requests})", dataset = "hf",
samples.len() samples = samples.len(),
requested = num_requests,
"skipping dataset oversampling"
); );
} else if !samples.is_empty() { } else if !samples.is_empty() {
let original_len = samples.len(); let original_len = samples.len();
@@ -652,9 +683,11 @@ pub fn load_hf_dataset(
req.request_id = Some(format!("{request_id_prefix}{}", original_len + i)); req.request_id = Some(format!("{request_id_prefix}{}", original_len + i));
samples.push(req); samples.push(req);
} }
println!( tracing::info!(
"Oversampled HF dataset from {original_len} to {} total samples.", dataset = "hf",
samples.len() original_samples = original_len,
samples = samples.len(),
"oversampled dataset"
); );
} }
} }
+9 -6
View File
@@ -5,6 +5,7 @@ pub mod custom;
pub mod hf_dataset; pub mod hf_dataset;
pub mod multi_turn; pub mod multi_turn;
pub mod prefix_repetition; pub mod prefix_repetition;
mod progress;
pub mod random; pub mod random;
pub mod random_mm; pub mod random_mm;
pub mod random_rerank; pub mod random_rerank;
@@ -90,9 +91,10 @@ pub fn oversample_requests(
return; return;
} }
if no_oversample { if no_oversample {
println!( tracing::info!(
"Skipping oversampling. Total samples: {} (requested: {num_requests})", samples = requests.len(),
requests.len() requested = num_requests,
"skipping dataset oversampling"
); );
return; return;
} }
@@ -103,9 +105,10 @@ pub fn oversample_requests(
req.request_id = Some(format!("{request_id_prefix}{}", original_len + i)); req.request_id = Some(format!("{request_id_prefix}{}", original_len + i));
requests.push(req); requests.push(req);
} }
println!( tracing::info!(
"Oversampled requests from {original_len} to {} total samples.", original_samples = original_len,
requests.len() samples = requests.len(),
"oversampled dataset"
); );
} }
+4 -3
View File
@@ -445,9 +445,10 @@ pub fn load_sharegpt_multi_turn(
conv.conversation_id = format!("{request_id_prefix}conv-{}", original_len + i); conv.conversation_id = format!("{request_id_prefix}conv-{}", original_len + i);
conversations.push(conv); conversations.push(conv);
} }
println!( tracing::info!(
"Oversampled multi-turn conversations from {original_len} to {} total.", original_conversations = original_len,
conversations.len() conversations = conversations.len(),
"oversampled multi-turn conversations"
); );
} }
@@ -41,11 +41,13 @@ pub fn generate_prefix_repetition_dataset(
} }
let total = prompts_per_prefix * num_prefixes; let total = prompts_per_prefix * num_prefixes;
if total != num_requests { if total != num_requests {
println!( tracing::info!(
"prefix_repetition: generating {total} requests \ requested = num_requests,
({num_prefixes} prefixes x {prompts_per_prefix} prompts each; \ generated = total,
{} dropped to divide evenly)", prefixes = num_prefixes,
num_requests - total prompts_per_prefix,
dropped = num_requests - total,
"adjusted prefix-repetition request count"
); );
} }
+77
View File
@@ -0,0 +1,77 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::time::{Duration, Instant};
use indicatif::{ProgressBar, ProgressStyle};
const REPORT_INTERVAL: Duration = Duration::from_secs(10);
/// Reports row download progress to an interactive progress bar, or through
/// periodic tracing events when the progress bar is hidden on a non-TTY.
pub(super) struct RowDownloadReporter {
progress: ProgressBar,
next_report: Instant,
}
impl RowDownloadReporter {
/// Creates a reporter that emits non-TTY updates every 10 seconds.
pub fn new() -> Self {
let progress = ProgressBar::new(0);
progress.set_style(
ProgressStyle::with_template(
"{spinner:.green} Fetching rows [{bar:30.cyan/blue}] {pos}/{len}",
)
.unwrap()
.progress_chars("#>-"),
);
Self {
progress,
next_report: Instant::now() + REPORT_INTERVAL,
}
}
/// Updates the current row count and reports progress when due.
pub fn update(&mut self, rows: usize, total: u64) {
let rows = rows as u64;
let total = total.max(rows);
self.progress.set_length(total);
self.progress.set_position(rows);
if self.should_report(Instant::now()) {
tracing::info!(rows, total, "fetching dataset rows");
}
}
/// Clears the interactive progress bar after the download completes.
pub fn finish(self) {
self.progress.finish_and_clear();
}
fn should_report(&mut self, now: Instant) -> bool {
if !self.progress.is_hidden() || now < self.next_report {
return false;
}
self.next_report = now + REPORT_INTERVAL;
true
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hidden_reporter_uses_ten_second_deadline() {
let start = Instant::now();
let mut reporter = RowDownloadReporter {
progress: ProgressBar::hidden(),
next_report: start + REPORT_INTERVAL,
};
assert!(!reporter.should_report(start + Duration::from_secs(9)));
assert!(reporter.should_report(start + Duration::from_secs(10)));
assert!(!reporter.should_report(start + Duration::from_secs(19)));
assert!(reporter.should_report(start + Duration::from_secs(20)));
}
}
+6 -3
View File
@@ -49,9 +49,12 @@ pub fn generate_random_dataset(
let (input_low, input_high) = range_ratio.input_bounds(real_input_len); let (input_low, input_high) = range_ratio.input_bounds(real_input_len);
let (output_low, output_high) = range_ratio.output_bounds(output_len); let (output_low, output_high) = range_ratio.output_bounds(output_len);
if !range_ratio.is_fixed() { if !range_ratio.is_fixed() {
println!( tracing::info!(
"Sampling input_len from [{input_low}, {input_high}] and \ input_low,
output_len from [{output_low}, {output_high}]" input_high,
output_low,
output_high,
"sampling random request lengths"
); );
} }
+15 -9
View File
@@ -23,8 +23,10 @@ const DEFAULT_SHAREGPT_FILE: &str = "ShareGPT_V3_unfiltered_cleaned_split.json";
/// Download the default ShareGPT dataset from HuggingFace Hub. /// Download the default ShareGPT dataset from HuggingFace Hub.
/// Uses hf-hub's built-in cache — subsequent calls return the cached path instantly. /// Uses hf-hub's built-in cache — subsequent calls return the cached path instantly.
pub fn download_sharegpt_dataset() -> Result<String> { pub fn download_sharegpt_dataset() -> Result<String> {
println!( tracing::info!(
"Downloading ShareGPT dataset from {DEFAULT_SHAREGPT_REPO}/{DEFAULT_SHAREGPT_FILE} ..." repository = DEFAULT_SHAREGPT_REPO,
file = DEFAULT_SHAREGPT_FILE,
"downloading ShareGPT dataset"
); );
let repo = crate::hub::HubRepo::dataset(DEFAULT_SHAREGPT_REPO.to_string()); let repo = crate::hub::HubRepo::dataset(DEFAULT_SHAREGPT_REPO.to_string());
let path = repo.get(DEFAULT_SHAREGPT_FILE).map_err(|e| { let path = repo.get(DEFAULT_SHAREGPT_FILE).map_err(|e| {
@@ -33,7 +35,7 @@ pub fn download_sharegpt_dataset() -> Result<String> {
)) ))
})?; })?;
let path_str = path.to_string_lossy().to_string(); let path_str = path.to_string_lossy().to_string();
println!("ShareGPT dataset ready: {path_str}"); tracing::info!(dataset = "sharegpt", path = %path_str, "dataset is ready");
Ok(path_str) Ok(path_str)
} }
@@ -135,9 +137,11 @@ pub fn load_sharegpt_dataset(
// Oversample if dataset is smaller than requested // Oversample if dataset is smaller than requested
if samples.len() < num_requests { if samples.len() < num_requests {
if no_oversample { if no_oversample {
println!( tracing::info!(
"Skipping oversampling. Total samples: {} (requested: {num_requests})", dataset = "sharegpt",
samples.len() samples = samples.len(),
requested = num_requests,
"skipping dataset oversampling"
); );
} else if !samples.is_empty() { } else if !samples.is_empty() {
let needed = num_requests - samples.len(); let needed = num_requests - samples.len();
@@ -147,9 +151,11 @@ pub fn load_sharegpt_dataset(
req.request_id = Some(format!("{request_id_prefix}{}", original_len + i)); req.request_id = Some(format!("{request_id_prefix}{}", original_len + i));
samples.push(req); samples.push(req);
} }
println!( tracing::info!(
"Oversampled requests from {original_len} to {} total samples.", dataset = "sharegpt",
samples.len() original_samples = original_len,
samples = samples.len(),
"oversampled dataset"
); );
} }
} }
+22 -16
View File
@@ -8,6 +8,7 @@ use rand::seq::SliceRandom;
use rand::{Rng, SeedableRng}; use rand::{Rng, SeedableRng};
use super::SampleRequest; use super::SampleRequest;
use super::progress::RowDownloadReporter;
use crate::cli::SpeedBenchConfig; use crate::cli::SpeedBenchConfig;
use crate::error::{BenchError, Result}; use crate::error::{BenchError, Result};
use crate::tokenizer::TokenizerKind; use crate::tokenizer::TokenizerKind;
@@ -35,11 +36,11 @@ pub fn download_speed_bench(config: SpeedBenchConfig) -> Result<String> {
// Return cached file if it exists // Return cached file if it exists
if cache_path.exists() { if cache_path.exists() {
let path_str = cache_path.to_string_lossy().to_string(); let path_str = cache_path.to_string_lossy().to_string();
println!("SPEED-Bench ({config_name}) cached: {path_str}"); tracing::info!(config = config_name, path = %path_str, "using cached SPEED-Bench dataset");
return Ok(path_str); return Ok(path_str);
} }
println!("Downloading SPEED-Bench ({config_name}) from HuggingFace datasets-server..."); tracing::info!(config = config_name, "downloading SPEED-Bench dataset");
let client = reqwest::blocking::Client::builder() let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120)) .timeout(std::time::Duration::from_secs(120))
@@ -49,6 +50,7 @@ pub fn download_speed_bench(config: SpeedBenchConfig) -> Result<String> {
let mut all_rows: Vec<serde_json::Value> = Vec::new(); let mut all_rows: Vec<serde_json::Value> = Vec::new();
let mut offset = 0usize; let mut offset = 0usize;
let page_size = 100usize; let page_size = 100usize;
let mut progress = RowDownloadReporter::new();
loop { loop {
let url = format!( let url = format!(
@@ -116,15 +118,14 @@ pub fn download_speed_bench(config: SpeedBenchConfig) -> Result<String> {
let fetched = rows.len(); let fetched = rows.len();
offset += fetched; offset += fetched;
// Print progress
let total = data["num_rows_total"].as_u64().unwrap_or(0); let total = data["num_rows_total"].as_u64().unwrap_or(0);
eprint!("\r Fetched {offset}/{total} rows..."); progress.update(offset, total);
if fetched < page_size { if fetched < page_size {
break; break;
} }
} }
eprintln!(); // newline after progress progress.finish();
if all_rows.is_empty() { if all_rows.is_empty() {
return Err(BenchError::Config( return Err(BenchError::Config(
@@ -137,9 +138,11 @@ pub fn download_speed_bench(config: SpeedBenchConfig) -> Result<String> {
std::fs::write(&cache_path, &json_str)?; std::fs::write(&cache_path, &json_str)?;
let path_str = cache_path.to_string_lossy().to_string(); let path_str = cache_path.to_string_lossy().to_string();
println!( tracing::info!(
"SPEED-Bench ({config_name}): {} rows saved to {path_str}", config = config_name,
all_rows.len() rows = all_rows.len(),
path = %path_str,
"saved SPEED-Bench dataset"
); );
Ok(path_str) Ok(path_str)
} }
@@ -263,9 +266,11 @@ pub fn load_speed_bench_dataset(
// Oversample if needed // Oversample if needed
if samples.len() < num_requests { if samples.len() < num_requests {
if no_oversample { if no_oversample {
println!( tracing::info!(
"Skipping oversampling. Total samples: {} (requested: {num_requests})", dataset = "speed-bench",
samples.len() samples = samples.len(),
requested = num_requests,
"skipping dataset oversampling"
); );
} else if !samples.is_empty() { } else if !samples.is_empty() {
let original_len = samples.len(); let original_len = samples.len();
@@ -275,9 +280,11 @@ pub fn load_speed_bench_dataset(
req.request_id = Some(format!("{request_id_prefix}{}", original_len + i)); req.request_id = Some(format!("{request_id_prefix}{}", original_len + i));
samples.push(req); samples.push(req);
} }
println!( tracing::info!(
"Oversampled SPEED-Bench from {original_len} to {} total samples.", dataset = "speed-bench",
samples.len() original_samples = original_len,
samples = samples.len(),
"oversampled dataset"
); );
} }
} }
@@ -288,7 +295,6 @@ pub fn load_speed_bench_dataset(
)); ));
} }
// Print category distribution
let mut cat_counts: std::collections::HashMap<&str, usize> = std::collections::HashMap::new(); let mut cat_counts: std::collections::HashMap<&str, usize> = std::collections::HashMap::new();
for entry in &filtered[..filtered.len().min(samples.len())] { for entry in &filtered[..filtered.len().min(samples.len())] {
let cat = entry.get("category").and_then(|c| c.as_str()).unwrap_or("unknown"); let cat = entry.get("category").and_then(|c| c.as_str()).unwrap_or("unknown");
@@ -297,7 +303,7 @@ pub fn load_speed_bench_dataset(
let mut cats: Vec<_> = cat_counts.into_iter().collect(); let mut cats: Vec<_> = cat_counts.into_iter().collect();
cats.sort_by_key(|b| std::cmp::Reverse(b.1)); cats.sort_by_key(|b| std::cmp::Reverse(b.1));
let cat_str: Vec<String> = cats.iter().map(|(k, v)| format!("{k}:{v}")).collect(); let cat_str: Vec<String> = cats.iter().map(|(k, v)| format!("{k}:{v}")).collect();
println!("SPEED-Bench categories: {}", cat_str.join(", ")); tracing::info!(categories = %cat_str.join(", "), "computed SPEED-Bench category distribution");
Ok(samples) Ok(samples)
} }
+1 -1
View File
@@ -33,7 +33,7 @@ pub fn prepare_process() {
if let Ok(new) = rlimit::increase_nofile_limit(u64::MAX) if let Ok(new) = rlimit::increase_nofile_limit(u64::MAX)
&& new > 1024 && new > 1024
{ {
eprintln!("Open-file limit: {new}"); tracing::info!(soft_limit = new, "raised open-file limit");
} }
} }
+12
View File
@@ -19,7 +19,19 @@ struct Cli {
args: vllm_bench::BenchServeArgs, args: vllm_bench::BenchServeArgs,
} }
// TODO: unify the tracing subscriber used by different binaries.
fn init_tracing() {
let filter = tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info"));
let _ = tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(std::io::stderr)
.try_init();
}
fn main() -> anyhow::Result<()> { fn main() -> anyhow::Result<()> {
init_tracing();
let cli = Cli::parse(); let cli = Cli::parse();
vllm_bench::prepare_process(); vllm_bench::prepare_process();
+18 -16
View File
@@ -7,6 +7,22 @@ use crate::datasets::SampleRequest;
use crate::metrics::{BenchmarkMetrics, MultiTurnMetrics}; use crate::metrics::{BenchmarkMetrics, MultiTurnMetrics};
use crate::multi_turn::ConversationOutput; use crate::multi_turn::ConversationOutput;
fn log_failed_requests(outputs: &[RequestFuncOutput]) {
let failed_outputs: Vec<_> = outputs.iter().filter(|output| !output.success).collect();
if failed_outputs.is_empty() {
return;
}
tracing::warn!(
failed_requests = failed_outputs.len(),
displayed_errors = failed_outputs.len().min(10),
"benchmark requests failed"
);
for (index, output) in failed_outputs.into_iter().take(10).enumerate() {
tracing::warn!(index, error = %output.error, "benchmark request failed");
}
}
/// Calculate benchmark metrics from request outputs. /// Calculate benchmark metrics from request outputs.
/// ///
/// Mirrors Python's `calculate_metrics()` from serve.py:392-599. /// Mirrors Python's `calculate_metrics()` from serve.py:392-599.
@@ -63,14 +79,7 @@ pub fn calculate_metrics(
let failed = outputs.len() - completed; let failed = outputs.len() - completed;
// Print failed request errors (capped to 10) log_failed_requests(outputs);
let failed_outputs: Vec<&RequestFuncOutput> = outputs.iter().filter(|o| !o.success).collect();
if !failed_outputs.is_empty() {
eprintln!("Failed requests during benchmark run detected (capping to 10):");
for (i, err) in failed_outputs.iter().take(10).enumerate() {
eprintln!("Error {i}: {}", err.error);
}
}
// Calculate max output tokens per second and max concurrent requests // Calculate max output tokens per second and max concurrent requests
let mut max_output_tokens_per_s = 0.0_f64; let mut max_output_tokens_per_s = 0.0_f64;
@@ -295,14 +304,7 @@ pub fn calculate_embedding_metrics(
let failed = outputs.len() - completed; let failed = outputs.len() - completed;
// Print failed request errors (capped to 10) log_failed_requests(outputs);
let failed_outputs: Vec<&RequestFuncOutput> = outputs.iter().filter(|o| !o.success).collect();
if !failed_outputs.is_empty() {
eprintln!("Failed requests during benchmark run detected (capping to 10):");
for (i, err) in failed_outputs.iter().take(10).enumerate() {
eprintln!("Error {i}: {}", err.error);
}
}
// Compute peak concurrent requests from start_time + latency windows // Compute peak concurrent requests from start_time + latency windows
let successful_outputs: Vec<&RequestFuncOutput> = let successful_outputs: Vec<&RequestFuncOutput> =
+50 -37
View File
@@ -6,6 +6,7 @@ use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use indicatif::{ProgressBar, ProgressStyle}; use indicatif::{ProgressBar, ProgressStyle};
use thiserror_ext::AsReport as _;
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
use crate::backends::{Backend, RequestFuncInput, RequestFuncOutput, get_backend}; use crate::backends::{Backend, RequestFuncInput, RequestFuncOutput, get_backend};
@@ -72,9 +73,13 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
let (model_id, model_name) = if let Some(ref m) = config.model { let (model_id, model_name) = if let Some(ref m) = config.model {
(m.clone(), config.model_name.clone()) (m.clone(), config.model_name.clone())
} else { } else {
println!("Model not specified, fetching first model from server..."); tracing::info!(base_url = %config.base_url, "fetching first model from server");
let (name, id) = get_first_model(&config.base_url, &client, &config.extra_headers).await?; let (name, id) = get_first_model(&config.base_url, &client, &config.extra_headers).await?;
println!("First model name: {name}, first model id: {id}"); tracing::info!(
model_name = name,
model_id = id,
"selected first model from server"
);
(id, Some(name)) (id, Some(name))
}; };
@@ -83,15 +88,18 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
None None
} else { } else {
let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id); let tid = config.tokenizer_id.as_deref().unwrap_or(&model_id);
println!("Loading tokenizer: {tid}"); tracing::info!(tokenizer = tid, "loading tokenizer");
let server_info = Some((config.base_url.as_str(), model_id.as_str())); let server_info = Some((config.base_url.as_str(), model_id.as_str()));
let t = crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info)?; let t = crate::tokenizer::load_tokenizer(tid, config.trust_remote_code, server_info)?;
println!("Tokenizer loaded successfully.");
Some(t) Some(t)
}; };
// Generate/load conversations // Generate/load conversations
println!("Generating multi-turn conversations..."); tracing::info!(
dataset = ?config.dataset_name,
conversations = config.num_prompts,
"generating multi-turn conversations"
);
let gen_start = Instant::now(); let gen_start = Instant::now();
let mut conversations = match config.dataset_name { let mut conversations = match config.dataset_name {
@@ -179,8 +187,11 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
let (filtered_conversations, filtered_turns) = let (filtered_conversations, filtered_turns) =
filter_turns_by_max_model_len(&mut conversations, max_model_len, no_history); filter_turns_by_max_model_len(&mut conversations, max_model_len, no_history);
if filtered_turns > 0 || filtered_conversations > 0 { if filtered_turns > 0 || filtered_conversations > 0 {
println!( tracing::info!(
"Filtered {filtered_turns} turn(s) and {filtered_conversations} conversation(s) above --max-model-len {max_model_len}." filtered_turns,
filtered_conversations,
max_model_len,
"filtered conversations above maximum model length"
); );
} }
if conversations.is_empty() { if conversations.is_empty() {
@@ -192,11 +203,11 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
let gen_elapsed = gen_start.elapsed(); let gen_elapsed = gen_start.elapsed();
let total_turns: usize = conversations.iter().map(|c| c.turns.len()).sum(); let total_turns: usize = conversations.iter().map(|c| c.turns.len()).sum();
println!( tracing::info!(
"Generated {} conversations ({} total turns) in {:.2}s", conversations = conversations.len(),
conversations.len(),
total_turns, total_turns,
gen_elapsed.as_secs_f64() elapsed_seconds = gen_elapsed.as_secs_f64(),
"generated multi-turn conversations"
); );
// Log prefix sharing info // Log prefix sharing info
@@ -208,18 +219,15 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
let conv_tokens = let conv_tokens =
(real_input_len as f64 * config.multi_turn_prefix_conversation_ratio).floor() as usize; (real_input_len as f64 * config.multi_turn_prefix_conversation_ratio).floor() as usize;
let unique_tokens = real_input_len.saturating_sub(global_tokens + conv_tokens); let unique_tokens = real_input_len.saturating_sub(global_tokens + conv_tokens);
println!( tracing::info!(
"User message prefix sharing: {:.0}% global ({} tokens), {:.0}% per-conversation ({} tokens), {:.0}% unique ({} tokens)", global_ratio = config.multi_turn_prefix_global_ratio,
config.multi_turn_prefix_global_ratio * 100.0,
global_tokens, global_tokens,
config.multi_turn_prefix_conversation_ratio * 100.0, conversation_ratio = config.multi_turn_prefix_conversation_ratio,
conv_tokens, conversation_tokens = conv_tokens,
(1.0 - config.multi_turn_prefix_global_ratio
- config.multi_turn_prefix_conversation_ratio)
* 100.0,
unique_tokens, unique_tokens,
history_accumulation = false,
"configured multi-turn prefix sharing"
); );
println!("No history accumulation: each turn sends fixed-length prompt only.");
} }
if config.dry_run { if config.dry_run {
@@ -253,7 +261,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
..Default::default() ..Default::default()
}; };
println!("Starting initial single prompt test run..."); tracing::info!("starting initial single-prompt test run");
let test_output = crate::ready_checker::wait_for_endpoint( let test_output = crate::ready_checker::wait_for_endpoint(
config.backend, config.backend,
&client, &client,
@@ -268,7 +276,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
test_output.error test_output.error
))); )));
} }
println!("Initial test run completed."); tracing::info!("initial single-prompt test run completed");
} }
// For random datasets in multi-turn mode, auto-set min_tokens to enforce // For random datasets in multi-turn mode, auto-set min_tokens to enforce
@@ -283,9 +291,10 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
"min_tokens".to_string(), "min_tokens".to_string(),
serde_json::json!(config.random_output_len), serde_json::json!(config.random_output_len),
); );
println!( tracing::info!(
"Auto-setting min_tokens={} for multi-turn random dataset (use --extra-body to override)", min_tokens = config.random_output_len,
config.random_output_len dataset = "random",
"set minimum output tokens for multi-turn dataset"
); );
} }
Some(body) Some(body)
@@ -297,7 +306,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
let spec_decode_before = let spec_decode_before =
fetch_spec_decode_metrics(&config.base_url, &client, &config.extra_headers).await; fetch_spec_decode_metrics(&config.base_url, &client, &config.extra_headers).await;
if spec_decode_before.is_some() { if spec_decode_before.is_some() {
println!("Speculative decoding detected, will collect metrics."); tracing::info!("detected speculative decoding; collecting metrics");
} }
// Start profiler if requested (immediate mode — no batch threshold) // Start profiler if requested (immediate mode — no batch threshold)
@@ -330,10 +339,13 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
}; };
// Main benchmark // Main benchmark
println!("Starting multi-turn benchmark..."); tracing::info!(
println!("Conversations: {}", conversations.len()); conversations = conversations.len(),
println!("Concurrency: {concurrency}"); total_turns,
println!("Inter-turn delay: {} ms", config.multi_turn_delay_ms); concurrency,
inter_turn_delay_ms = config.multi_turn_delay_ms,
"starting multi-turn benchmark"
);
let max_turn_count = conversations.iter().map(|c| c.turns.len()).max().unwrap_or(0); let max_turn_count = conversations.iter().map(|c| c.turns.len()).max().unwrap_or(0);
@@ -364,11 +376,12 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
); );
if let Some(modules) = config.lora_modules.as_ref() { if let Some(modules) = config.lora_modules.as_ref() {
let names: Vec<&str> = modules.iter().map(|s| s.as_ref()).collect(); let names: Vec<&str> = modules.iter().map(|s| s.as_ref()).collect();
println!( tracing::info!(
"LoRA adapters ({}): {:?} [assignment={:?}, scope=conversation]", adapters = modules.len(),
modules.len(), names = ?names,
names, assignment = ?config.lora_assignment,
config.lora_assignment scope = "conversation",
"assigned LoRA adapters"
); );
} }
@@ -433,7 +446,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
match handle.await { match handle.await {
Ok(output) => all_outputs.push(output), Ok(output) => all_outputs.push(output),
Err(e) => { Err(e) => {
eprintln!("Conversation task panicked: {e}"); tracing::error!(error = %e.as_report(), "conversation task panicked");
} }
} }
} }
@@ -453,7 +466,7 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
if let Some((cancel_tx, task)) = profile_task { if let Some((cancel_tx, task)) = profile_task {
let _ = cancel_tx.send(()); let _ = cancel_tx.send(());
if let Err(e) = task.await { if let Err(e) = task.await {
eprintln!("WARNING: Profile background task failed: {e}"); tracing::error!(error = %e.as_report(), "profiler background task failed");
} }
} }
+2 -2
View File
@@ -603,7 +603,7 @@ fn add_metric_stats(
pub fn save_result(json: &Value, file_path: &str) -> Result<()> { pub fn save_result(json: &Value, file_path: &str) -> Result<()> {
let content = serde_json::to_string(json)?; let content = serde_json::to_string(json)?;
std::fs::write(file_path, content)?; std::fs::write(file_path, content)?;
println!("Results saved to {file_path}"); tracing::info!(path = file_path, "saved benchmark results");
Ok(()) Ok(())
} }
@@ -618,7 +618,7 @@ pub fn append_result(json: &Value, file_path: &str) -> Result<()> {
file.write_all(b"\n")?; file.write_all(b"\n")?;
} }
file.write_all(content.as_bytes())?; file.write_all(content.as_bytes())?;
println!("Results appended to {file_path}"); tracing::info!(path = file_path, "appended benchmark results");
Ok(()) Ok(())
} }
+8 -2
View File
@@ -23,7 +23,11 @@ pub async fn wait_for_endpoint(
let backend = get_backend(backend)?; let backend = get_backend(backend)?;
let deadline = Instant::now() + std::time::Duration::from_secs(timeout_seconds); let deadline = Instant::now() + std::time::Duration::from_secs(timeout_seconds);
println!("Waiting for endpoint to become up in {timeout_seconds}s"); tracing::info!(
timeout_seconds,
retry_interval,
"waiting for endpoint readiness"
);
let pb = ProgressBar::new(timeout_seconds); let pb = ProgressBar::new(timeout_seconds);
pb.set_style( pb.set_style(
@@ -53,7 +57,9 @@ pub async fn wait_for_endpoint(
Ok(output) => { Ok(output) => {
let err = output.error.clone(); let err = output.error.clone();
let err_last_line = err.lines().last().unwrap_or(&err); let err_last_line = err.lines().last().unwrap_or(&err);
eprintln!("Endpoint is not ready. Error='{err_last_line}'"); pb.suspend(|| {
tracing::warn!(error = err_last_line, "endpoint is not ready");
});
last_error = err; last_error = err;
} }
Err(e) => { Err(e) => {
+1 -1
View File
@@ -16,7 +16,7 @@ async fn reset_prefix_cache(base_url: &str) -> Result<()> {
.await .await
.map_err(|e| BenchError::Backend(format!("Failed to reset prefix cache: {e}")))?; .map_err(|e| BenchError::Backend(format!("Failed to reset prefix cache: {e}")))?;
if resp.status().is_success() { if resp.status().is_success() {
println!("Prefix cache reset successfully."); tracing::info!(url = %url, "reset prefix cache");
} else { } else {
let status = resp.status(); let status = resp.status();
let body = resp.text().await.unwrap_or_default(); let body = resp.text().await.unwrap_or_default();
+15 -9
View File
@@ -199,7 +199,12 @@ pub fn load_builtin_tiktoken(encoding: &str) -> Result<TiktokenTokenizer> {
} }
}; };
let bpe = bpe.map_err(|e| BenchError::Tokenizer(format!("Failed to load {encoding}: {e}")))?; let bpe = bpe.map_err(|e| BenchError::Tokenizer(format!("Failed to load {encoding}: {e}")))?;
println!("Tokenizer: Built-in tiktoken {encoding} (vocab_size={vocab_size})"); tracing::info!(
encoding,
kind = "built-in-tiktoken",
vocab_size,
"loaded tokenizer"
);
Ok(TiktokenTokenizer::from_builtin_bpe(bpe, vocab_size)) Ok(TiktokenTokenizer::from_builtin_bpe(bpe, vocab_size))
} }
@@ -309,15 +314,16 @@ fn build_tiktoken(
} }
} }
println!( tracing::info!(
"Loading tiktoken model for '{model_id}' (base={}, special={}, pat={})...", model = model_id,
num_base_tokens, base_tokens = num_base_tokens,
all_special_tokens.len(), special_tokens = all_special_tokens.len(),
if pattern.is_some() { pattern = if pattern.is_some() {
"custom" "custom"
} else { } else {
"default" "default"
}, },
"loading tiktoken model"
); );
TiktokenTokenizer::from_file( TiktokenTokenizer::from_file(
@@ -438,9 +444,9 @@ fn extract_pat_str_from_source(source: &str) -> Option<String> {
if !fragments.is_empty() { if !fragments.is_empty() {
let pattern = fragments.join("|"); let pattern = fragments.join("|");
println!( tracing::debug!(
"Extracted pat_str from Python source: {} fragments", fragments = fragments.len(),
fragments.len() "extracted tiktoken pattern from Python source"
); );
return Some(pattern); return Some(pattern);
} }
+27 -9
View File
@@ -4,6 +4,7 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::path::Path; use std::path::Path;
use thiserror_ext::AsReport as _;
use tokenizers::Tokenizer; use tokenizers::Tokenizer;
use crate::error::{BenchError, Result}; use crate::error::{BenchError, Result};
@@ -214,29 +215,46 @@ pub fn load_tokenizer(
// 1. Try local HuggingFace tokenizer (tokenizer.json) // 1. Try local HuggingFace tokenizer (tokenizer.json)
match try_load_local(model_id) { match try_load_local(model_id) {
Ok(tok) => { Ok(tok) => {
println!("Tokenizer: Local (vocab_size={})", tok.get_vocab_size(true)); tracing::info!(
model = model_id,
kind = "local",
vocab_size = tok.get_vocab_size(true),
"loaded tokenizer"
);
Ok(TokenizerKind::Local(Box::new(tok))) Ok(TokenizerKind::Local(Box::new(tok)))
} }
Err(local_err) => { Err(local_err) => {
// 2. Try tiktoken format // 2. Try tiktoken format
println!("No tokenizer.json for '{model_id}', trying tiktoken format..."); tracing::info!(
model = model_id,
error = %local_err.as_report(),
"local tokenizer unavailable; trying tiktoken"
);
match crate::tiktoken::try_load_tiktoken(model_id) { match crate::tiktoken::try_load_tiktoken(model_id) {
Ok(tok) => { Ok(tok) => {
println!("Tokenizer: Tiktoken (vocab_size={})", tok.vocab_size()); tracing::info!(
model = model_id,
kind = "tiktoken",
vocab_size = tok.vocab_size(),
"loaded tokenizer"
);
Ok(TokenizerKind::Tiktoken(tok)) Ok(TokenizerKind::Tiktoken(tok))
} }
Err(tiktoken_err) => { Err(tiktoken_err) => {
// 3. Try server-side fallback // 3. Try server-side fallback
if let Some((base_url, model)) = server_info { if let Some((base_url, model)) = server_info {
println!( tracing::info!(
"Tiktoken also not available ({tiktoken_err}), \ model = model_id,
trying server-side tokenization..." error = %tiktoken_err.as_report(),
"tiktoken unavailable; trying server-side tokenization"
); );
match ServerTokenizer::new(base_url, model) { match ServerTokenizer::new(base_url, model) {
Ok(srv) => { Ok(srv) => {
println!( tracing::info!(
"Tokenizer: Server (vocab_size≈{})", model = model_id,
srv.cached_vocab_size kind = "server",
vocab_size = srv.cached_vocab_size,
"loaded tokenizer"
); );
return Ok(TokenizerKind::Server(srv)); return Ok(TokenizerKind::Server(srv));
} }
+17 -7
View File
@@ -26,18 +26,21 @@ const RESET: &str = "\x1b[0m";
const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] = const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
format_description!("[month]-[day] [hour]:[minute]:[second]"); format_description!("[month]-[day] [hour]:[minute]:[second]");
const PROCESS_LABEL: &str = "RustFrontend";
/// Install the process-wide vLLM-style tracing subscriber for the CLI binary. /// Install the process-wide vLLM-style tracing subscriber for the CLI binary.
pub(crate) fn init_tracing() { pub(crate) fn init_tracing(process_label: &str) {
let filter = build_targets_filter( let filter = build_targets_filter(
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(), env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
env::var("RUST_LOG").ok().as_deref(), env::var("RUST_LOG").ok().as_deref(),
); );
let formatter = VllmEventFormatter::new(); let formatter = VllmEventFormatter::new(process_label);
let _ = tracing_subscriber::registry() let _ = tracing_subscriber::registry()
.with(tracing_subscriber::fmt::layer().event_format(formatter).with_filter(filter)) .with(
tracing_subscriber::fmt::layer()
.event_format(formatter)
.with_writer(std::io::stderr)
.with_filter(filter),
)
.try_init(); .try_init();
} }
@@ -94,9 +97,9 @@ struct VllmEventFormatter {
} }
impl VllmEventFormatter { impl VllmEventFormatter {
fn new() -> Self { fn new(process_label: &str) -> Self {
Self { Self {
prefix: format!("({} pid={})", PROCESS_LABEL, process::id()), prefix: format!("({process_label} pid={})", process::id()),
timer: VllmLocalTimer::default(), timer: VllmLocalTimer::default(),
} }
} }
@@ -291,6 +294,13 @@ fn map_python_log_level(level: &str) -> LevelFilter {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn formatter_prefix_uses_process_label() {
let formatter = VllmEventFormatter::new("Bench");
assert_eq!(formatter.prefix, format!("(Bench pid={})", process::id()));
}
#[test] #[test]
fn rust_log_target_overrides_are_merged_with_vllm_default_level() { fn rust_log_target_overrides_are_merged_with_vllm_default_level() {
let filter = build_targets_filter(Some("DEBUG"), Some("hyper=warn,tower=error")); let filter = build_targets_filter(Some("DEBUG"), Some("hyper=warn,tower=error"));
+9 -1
View File
@@ -5,6 +5,7 @@ mod cli;
mod logging; mod logging;
use std::env; use std::env;
use std::ffi::OsStr;
use std::process::ExitStatus; use std::process::ExitStatus;
use anyhow::{Context, Result, anyhow, bail}; use anyhow::{Context, Result, anyhow, bail};
@@ -82,7 +83,14 @@ fn shutdown_signal() -> CancellationToken {
} }
fn main() -> Result<()> { fn main() -> Result<()> {
logging::init_tracing(); let process_label =
match env::args_os().nth(1).as_deref().and_then(OsStr::to_str).unwrap_or_default() {
"bench" => "Bench",
"serve" | "frontend" => "RustFrontend",
_ => "Rust",
};
logging::init_tracing(process_label);
let cli = Cli::parse(); let cli = Cli::parse();
let mut runtime = tokio::runtime::Builder::new_multi_thread(); let mut runtime = tokio::runtime::Builder::new_multi_thread();