[Rust Frontend] Improve startup failure and readiness logs (#50406)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
45b60e3914
commit
e9096fcb12
@@ -5,11 +5,12 @@ mod cli;
|
||||
|
||||
use std::env;
|
||||
use std::ffi::OsStr;
|
||||
use std::process::ExitStatus;
|
||||
use std::process::{ExitCode, ExitStatus};
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use tracing::{error, info, warn};
|
||||
use vllm_managed_engine::ManagedEngineHandle;
|
||||
|
||||
use crate::cli::{BenchCommand, Cli, Command};
|
||||
@@ -81,7 +82,7 @@ fn shutdown_signal() -> CancellationToken {
|
||||
token
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
fn main() -> ExitCode {
|
||||
let process_label =
|
||||
match env::args_os().nth(1).as_deref().and_then(OsStr::to_str).unwrap_or_default() {
|
||||
"bench" => "Bench",
|
||||
@@ -98,10 +99,18 @@ fn main() -> Result<()> {
|
||||
runtime.worker_threads(worker_threads);
|
||||
}
|
||||
|
||||
runtime
|
||||
let result = runtime
|
||||
.build()
|
||||
.context("failed to build Tokio runtime")?
|
||||
.block_on(async_main(cli))
|
||||
.context("failed to build Tokio runtime")
|
||||
.and_then(|runtime| runtime.block_on(async_main(cli)));
|
||||
|
||||
match result {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(error) => {
|
||||
error!("process failed with error: {:#?}", error.as_report());
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn async_main(cli: Cli) -> Result<()> {
|
||||
|
||||
@@ -181,12 +181,15 @@ where
|
||||
result = build_state(&config) => result?,
|
||||
_ = shutdown.cancelled() => return Ok(()),
|
||||
};
|
||||
let model = state.primary_model_name().to_owned();
|
||||
let app = extend_router(build_router(state.clone()));
|
||||
|
||||
info!(model, "starting vLLM server");
|
||||
|
||||
let listener = Listener::bind(&config.listener_mode)
|
||||
.await
|
||||
.context("failed to bind listener for OpenAI server")?;
|
||||
let bind_address = listener.local_addr_display()?;
|
||||
let model = state.primary_model_name().to_owned();
|
||||
let app = extend_router(build_router(state.clone()));
|
||||
|
||||
// Optionally bind the gRPC Inference server on a separate port. Bind
|
||||
// synchronously here so bind errors (port in use, permission denied, ...)
|
||||
@@ -220,8 +223,14 @@ where
|
||||
.add_service(health_service)
|
||||
.add_service(control_service)
|
||||
.add_service(inference_service);
|
||||
info!(%addr, tls = grpc_tls.is_some(), "starting gRPC server");
|
||||
Some((grpc_listener, svc, grpc_tls, health_reporter, engine_health))
|
||||
Some((
|
||||
addr,
|
||||
grpc_listener,
|
||||
svc,
|
||||
grpc_tls,
|
||||
health_reporter,
|
||||
engine_health,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -231,7 +240,7 @@ where
|
||||
} else {
|
||||
"http"
|
||||
};
|
||||
info!(%bind_address, %scheme, %model, "starting OpenAI server");
|
||||
let model = model.as_str();
|
||||
|
||||
// Run HTTP and gRPC concurrently under a child token of the caller's shutdown
|
||||
// token. Caller cancellation propagates into both protocols; if either
|
||||
@@ -285,6 +294,11 @@ where
|
||||
};
|
||||
let server = serve_connections(listener, app, shutdown.cancelled_owned(), timeouts);
|
||||
|
||||
info!(
|
||||
bind_address,
|
||||
scheme, model, "OpenAI server is ready to accept requests"
|
||||
);
|
||||
|
||||
let result = tokio::select! {
|
||||
result = server => {
|
||||
result.context("HTTP server failed")
|
||||
@@ -305,13 +319,15 @@ where
|
||||
let server_shutdown = server_shutdown.clone();
|
||||
let force_shutdown = force_shutdown.clone();
|
||||
async move {
|
||||
let Some((grpc_listener, svc, grpc_tls, health_reporter, engine_health)) = grpc_setup
|
||||
let Some((addr, grpc_listener, svc, grpc_tls, health_reporter, engine_health)) =
|
||||
grpc_setup
|
||||
else {
|
||||
// No gRPC configured: just wait for shutdown so we do not race the
|
||||
// join! by resolving early and tripping the cancellation token.
|
||||
shutdown.cancelled().await;
|
||||
return Ok(());
|
||||
};
|
||||
let tls = grpc_tls.is_some();
|
||||
let incoming = match grpc_tls {
|
||||
Some(context) => MaybeTlsListener::tls(grpc_listener, context),
|
||||
None => MaybeTlsListener::plain(grpc_listener),
|
||||
@@ -320,6 +336,8 @@ where
|
||||
svc.serve_with_incoming_shutdown(incoming, shutdown.clone().cancelled_owned());
|
||||
let health_monitor = grpc::monitor_health(health_reporter, engine_health, shutdown);
|
||||
|
||||
info!(%addr, tls, model, "gRPC server is ready to accept requests");
|
||||
|
||||
let server = async move {
|
||||
let result = tokio::select! {
|
||||
result = server => {
|
||||
|
||||
Reference in New Issue
Block a user