[Rust Frontend] Improve startup failure and readiness logs (#50406)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Bugen Zhao
2026-07-30 14:18:17 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 45b60e3914
commit e9096fcb12
2 changed files with 39 additions and 12 deletions
+15 -6
View File
@@ -5,11 +5,12 @@ mod cli;
use std::env;
use std::ffi::OsStr;
use std::process::ExitStatus;
use std::process::{ExitCode, ExitStatus};
use anyhow::{Context, Result, anyhow, bail};
use thiserror_ext::AsReport as _;
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use tracing::{error, info, warn};
use vllm_managed_engine::ManagedEngineHandle;
use crate::cli::{BenchCommand, Cli, Command};
@@ -81,7 +82,7 @@ fn shutdown_signal() -> CancellationToken {
token
}
fn main() -> Result<()> {
fn main() -> ExitCode {
let process_label =
match env::args_os().nth(1).as_deref().and_then(OsStr::to_str).unwrap_or_default() {
"bench" => "Bench",
@@ -98,10 +99,18 @@ fn main() -> Result<()> {
runtime.worker_threads(worker_threads);
}
runtime
let result = runtime
.build()
.context("failed to build Tokio runtime")?
.block_on(async_main(cli))
.context("failed to build Tokio runtime")
.and_then(|runtime| runtime.block_on(async_main(cli)));
match result {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
error!("process failed with error: {:#?}", error.as_report());
ExitCode::FAILURE
}
}
}
async fn async_main(cli: Cli) -> Result<()> {
+24 -6
View File
@@ -181,12 +181,15 @@ where
result = build_state(&config) => result?,
_ = shutdown.cancelled() => return Ok(()),
};
let model = state.primary_model_name().to_owned();
let app = extend_router(build_router(state.clone()));
info!(model, "starting vLLM server");
let listener = Listener::bind(&config.listener_mode)
.await
.context("failed to bind listener for OpenAI server")?;
let bind_address = listener.local_addr_display()?;
let model = state.primary_model_name().to_owned();
let app = extend_router(build_router(state.clone()));
// Optionally bind the gRPC Inference server on a separate port. Bind
// synchronously here so bind errors (port in use, permission denied, ...)
@@ -220,8 +223,14 @@ where
.add_service(health_service)
.add_service(control_service)
.add_service(inference_service);
info!(%addr, tls = grpc_tls.is_some(), "starting gRPC server");
Some((grpc_listener, svc, grpc_tls, health_reporter, engine_health))
Some((
addr,
grpc_listener,
svc,
grpc_tls,
health_reporter,
engine_health,
))
} else {
None
};
@@ -231,7 +240,7 @@ where
} else {
"http"
};
info!(%bind_address, %scheme, %model, "starting OpenAI server");
let model = model.as_str();
// Run HTTP and gRPC concurrently under a child token of the caller's shutdown
// token. Caller cancellation propagates into both protocols; if either
@@ -285,6 +294,11 @@ where
};
let server = serve_connections(listener, app, shutdown.cancelled_owned(), timeouts);
info!(
bind_address,
scheme, model, "OpenAI server is ready to accept requests"
);
let result = tokio::select! {
result = server => {
result.context("HTTP server failed")
@@ -305,13 +319,15 @@ where
let server_shutdown = server_shutdown.clone();
let force_shutdown = force_shutdown.clone();
async move {
let Some((grpc_listener, svc, grpc_tls, health_reporter, engine_health)) = grpc_setup
let Some((addr, grpc_listener, svc, grpc_tls, health_reporter, engine_health)) =
grpc_setup
else {
// No gRPC configured: just wait for shutdown so we do not race the
// join! by resolving early and tripping the cancellation token.
shutdown.cancelled().await;
return Ok(());
};
let tls = grpc_tls.is_some();
let incoming = match grpc_tls {
Some(context) => MaybeTlsListener::tls(grpc_listener, context),
None => MaybeTlsListener::plain(grpc_listener),
@@ -320,6 +336,8 @@ where
svc.serve_with_incoming_shutdown(incoming, shutdown.clone().cancelled_owned());
let health_monitor = grpc::monitor_health(health_reporter, engine_health, shutdown);
info!(%addr, tls, model, "gRPC server is ready to accept requests");
let server = async move {
let result = tokio::select! {
result = server => {