diff --git a/rust/src/cmd/src/main.rs b/rust/src/cmd/src/main.rs index 86c9ab6f93..4e92ece709 100644 --- a/rust/src/cmd/src/main.rs +++ b/rust/src/cmd/src/main.rs @@ -5,11 +5,12 @@ mod cli; use std::env; use std::ffi::OsStr; -use std::process::ExitStatus; +use std::process::{ExitCode, ExitStatus}; use anyhow::{Context, Result, anyhow, bail}; +use thiserror_ext::AsReport as _; use tokio_util::sync::CancellationToken; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use vllm_managed_engine::ManagedEngineHandle; use crate::cli::{BenchCommand, Cli, Command}; @@ -81,7 +82,7 @@ fn shutdown_signal() -> CancellationToken { token } -fn main() -> Result<()> { +fn main() -> ExitCode { let process_label = match env::args_os().nth(1).as_deref().and_then(OsStr::to_str).unwrap_or_default() { "bench" => "Bench", @@ -98,10 +99,18 @@ fn main() -> Result<()> { runtime.worker_threads(worker_threads); } - runtime + let result = runtime .build() - .context("failed to build Tokio runtime")? - .block_on(async_main(cli)) + .context("failed to build Tokio runtime") + .and_then(|runtime| runtime.block_on(async_main(cli))); + + match result { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + error!("process failed with error: {:#?}", error.as_report()); + ExitCode::FAILURE + } + } } async fn async_main(cli: Cli) -> Result<()> { diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 161c0f8ef4..224ec672ae 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -181,12 +181,15 @@ where result = build_state(&config) => result?, _ = shutdown.cancelled() => return Ok(()), }; + let model = state.primary_model_name().to_owned(); + let app = extend_router(build_router(state.clone())); + + info!(model, "starting vLLM server"); + let listener = Listener::bind(&config.listener_mode) .await .context("failed to bind listener for OpenAI server")?; let bind_address = listener.local_addr_display()?; - let model = state.primary_model_name().to_owned(); - let app = extend_router(build_router(state.clone())); // Optionally bind the gRPC Inference server on a separate port. Bind // synchronously here so bind errors (port in use, permission denied, ...) @@ -220,8 +223,14 @@ where .add_service(health_service) .add_service(control_service) .add_service(inference_service); - info!(%addr, tls = grpc_tls.is_some(), "starting gRPC server"); - Some((grpc_listener, svc, grpc_tls, health_reporter, engine_health)) + Some(( + addr, + grpc_listener, + svc, + grpc_tls, + health_reporter, + engine_health, + )) } else { None }; @@ -231,7 +240,7 @@ where } else { "http" }; - info!(%bind_address, %scheme, %model, "starting OpenAI server"); + let model = model.as_str(); // Run HTTP and gRPC concurrently under a child token of the caller's shutdown // token. Caller cancellation propagates into both protocols; if either @@ -285,6 +294,11 @@ where }; let server = serve_connections(listener, app, shutdown.cancelled_owned(), timeouts); + info!( + bind_address, + scheme, model, "OpenAI server is ready to accept requests" + ); + let result = tokio::select! { result = server => { result.context("HTTP server failed") @@ -305,13 +319,15 @@ where let server_shutdown = server_shutdown.clone(); let force_shutdown = force_shutdown.clone(); async move { - let Some((grpc_listener, svc, grpc_tls, health_reporter, engine_health)) = grpc_setup + let Some((addr, grpc_listener, svc, grpc_tls, health_reporter, engine_health)) = + grpc_setup else { // No gRPC configured: just wait for shutdown so we do not race the // join! by resolving early and tripping the cancellation token. shutdown.cancelled().await; return Ok(()); }; + let tls = grpc_tls.is_some(); let incoming = match grpc_tls { Some(context) => MaybeTlsListener::tls(grpc_listener, context), None => MaybeTlsListener::plain(grpc_listener), @@ -320,6 +336,8 @@ where svc.serve_with_incoming_shutdown(incoming, shutdown.clone().cancelled_owned()); let health_monitor = grpc::monitor_health(health_reporter, engine_health, shutdown); + info!(%addr, tls, model, "gRPC server is ready to accept requests"); + let server = async move { let result = tokio::select! { result = server => {