[Feature][Rust Frontend] Add Shutdown control RPC (#59316)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
b22494cc0c
commit
f28a508162
@@ -87,6 +87,7 @@ async fn main() -> Result<()> {
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: Duration::ZERO,
|
||||
manages_engine: false,
|
||||
keep_alive_timeout: Duration::from_secs(5),
|
||||
profiler: None,
|
||||
};
|
||||
|
||||
@@ -277,6 +277,9 @@ pub struct Config {
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
|
||||
pub shutdown_timeout: Duration,
|
||||
/// Whether the caller manages the engine process and shuts it down when
|
||||
/// the shutdown token is cancelled. Enables the gRPC `Control.Shutdown` RPC.
|
||||
pub manages_engine: bool,
|
||||
/// Maximum idle time on a keep-alive HTTP connection before the server
|
||||
/// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
|
||||
pub keep_alive_timeout: Duration,
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::sync::Arc;
|
||||
use serde_json::Value as JsonValue;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, Response, Status};
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
|
||||
@@ -23,6 +24,7 @@ pub(crate) type ControlGrpcService = ControlServer<ControlServiceImpl>;
|
||||
pub struct ControlServiceImpl {
|
||||
state: Arc<AppState>,
|
||||
rl_lock: Mutex<()>,
|
||||
engine_shutdown: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
impl ControlServiceImpl {
|
||||
@@ -30,9 +32,15 @@ impl ControlServiceImpl {
|
||||
Self {
|
||||
state,
|
||||
rl_lock: Mutex::new(()),
|
||||
engine_shutdown: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_engine_shutdown(mut self, shutdown: Option<CancellationToken>) -> Self {
|
||||
self.engine_shutdown = shutdown;
|
||||
self
|
||||
}
|
||||
|
||||
fn ready(&self) -> &EngineCoreReadyResponse {
|
||||
self.state.engine_core_client().ready_response()
|
||||
}
|
||||
@@ -179,6 +187,18 @@ fn list_loras_status(error: LoraDisabledError) -> Status {
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl pb::control_server::Control for ControlServiceImpl {
|
||||
async fn shutdown(
|
||||
&self,
|
||||
_request: Request<pb::ShutdownRequest>,
|
||||
) -> Result<Response<pb::ShutdownResponse>, Status> {
|
||||
let shutdown = self.engine_shutdown.as_ref().ok_or_else(|| {
|
||||
Status::failed_precondition("this frontend does not manage the engine process")
|
||||
})?;
|
||||
tracing::info!("engine shutdown requested via gRPC");
|
||||
shutdown.cancel();
|
||||
Ok(Response::new(pb::ShutdownResponse {}))
|
||||
}
|
||||
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
_request: Request<pb::GetServerInfoRequest>,
|
||||
|
||||
@@ -366,6 +366,7 @@ where
|
||||
engine_id,
|
||||
default_ready_response(),
|
||||
backend,
|
||||
None,
|
||||
move |dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
@@ -387,6 +388,7 @@ async fn setup_grpc_service_with_engine_script<F>(
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready: EngineCoreReadyResponse,
|
||||
backend: Arc<dyn ChatTextBackend>,
|
||||
engine_shutdown: Option<tokio_util::sync::CancellationToken>,
|
||||
script: F,
|
||||
) -> (
|
||||
InferenceServer<InferenceServiceImpl>,
|
||||
@@ -425,7 +427,7 @@ where
|
||||
(
|
||||
InferenceServer::new(InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES),
|
||||
ControlServer::new(ControlServiceImpl::new(state)),
|
||||
ControlServer::new(ControlServiceImpl::new(state).with_engine_shutdown(engine_shutdown)),
|
||||
engine_health,
|
||||
engine_task,
|
||||
)
|
||||
@@ -1920,6 +1922,7 @@ async fn control_reports_server_and_model_info() {
|
||||
b"engine-grpc-info".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
@@ -1987,6 +1990,7 @@ async fn control_lora_lifecycle_selects_adapter_for_generation() {
|
||||
b"engine-grpc-lora".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
reply_utility_bool(dealer, push, "add_lora", true).await;
|
||||
@@ -2111,6 +2115,7 @@ async fn control_list_loras_requires_lora_enabled_engine() {
|
||||
b"engine-grpc-lora-disabled".to_vec(),
|
||||
default_ready_response(),
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|_, _| boxed_test_future(async move {}),
|
||||
)
|
||||
.await;
|
||||
@@ -2143,6 +2148,7 @@ async fn control_forwards_weight_update_without_pause_guard() {
|
||||
b"engine-grpc-rl".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let frames = recv_engine_message(dealer).await;
|
||||
@@ -2392,9 +2398,9 @@ async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy()
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn grpc_health_watch_closes_on_graceful_shutdown() {
|
||||
async fn grpc_shutdown_requires_managed_engine() {
|
||||
let (inference_service, control_service, engine_health, _engine_task) = setup_grpc_service(
|
||||
b"engine-grpc-health-shutdown",
|
||||
b"engine-grpc-external-shutdown",
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
@@ -2406,6 +2412,47 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() {
|
||||
shutdown.clone(),
|
||||
)
|
||||
.await;
|
||||
let mut control_client = ControlClient::new(channel.clone());
|
||||
|
||||
let status = control_client
|
||||
.shutdown(pb::ShutdownRequest {})
|
||||
.await
|
||||
.expect_err("external engine shutdown must be rejected");
|
||||
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
|
||||
assert!(!shutdown.is_cancelled());
|
||||
|
||||
let health = HealthClient::new(channel)
|
||||
.check(HealthCheckRequest {
|
||||
service: "vllm.Inference".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("server remains healthy after rejected shutdown")
|
||||
.into_inner();
|
||||
assert_eq!(health.status, HealthServingStatus::Serving as i32);
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn grpc_shutdown_closes_health_watch_and_server() {
|
||||
let shutdown = tokio_util::sync::CancellationToken::new();
|
||||
let (inference_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service_with_engine_script(
|
||||
b"engine-grpc-health-shutdown",
|
||||
default_ready_response(),
|
||||
Arc::new(FakeTextBackend),
|
||||
Some(shutdown.clone()),
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
shutdown.child_token(),
|
||||
)
|
||||
.await;
|
||||
let mut control_client = ControlClient::new(channel.clone());
|
||||
let mut health_client = HealthClient::new(channel);
|
||||
let mut stream = health_client
|
||||
.watch(HealthCheckRequest {
|
||||
@@ -2426,7 +2473,14 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() {
|
||||
"unexpected initial health status for vllm.Inference"
|
||||
);
|
||||
|
||||
shutdown.cancel();
|
||||
control_client
|
||||
.shutdown(pb::ShutdownRequest {})
|
||||
.await
|
||||
.expect("accept managed engine shutdown");
|
||||
assert!(
|
||||
shutdown.is_cancelled(),
|
||||
"engine owner must receive shutdown"
|
||||
);
|
||||
|
||||
let update = tokio::time::timeout(Duration::from_secs(2), stream.message())
|
||||
.await
|
||||
|
||||
@@ -247,9 +247,11 @@ where
|
||||
let engine_health = state.engine_core_client().subscribe_health();
|
||||
health_reporter.set_serving::<grpc::InferenceGrpcService>().await;
|
||||
health_reporter.set_serving::<grpc::ControlGrpcService>().await;
|
||||
let control_service =
|
||||
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
let control_service = grpc::ControlGrpcService::new(
|
||||
grpc::ControlServiceImpl::new(state.clone())
|
||||
.with_engine_shutdown(config.manages_engine.then(|| shutdown.clone())),
|
||||
)
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
let inference_service =
|
||||
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
|
||||
Reference in New Issue
Block a user