From f28a5081629377c36cd219a35070555b72279109 Mon Sep 17 00:00:00 2001 From: JulienDarve <86800349+JulienDarve@users.noreply.github.com> Date: Wed, 30 Sep 2026 05:49:25 -0700 Subject: [PATCH] [Feature][Rust Frontend] Add Shutdown control RPC (#59316) Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: Julien Darve Signed-off-by: Bugen Zhao --- docs/usage/security.md | 2 +- rust/proto/README.md | 2 +- rust/proto/control.proto | 8 +++ rust/src/cmd/src/cli.rs | 6 ++ rust/src/cmd/src/cli/tests.rs | 26 ++++++++ .../examples/external_engine_openai_qwen.rs | 1 + rust/src/server/src/config.rs | 3 + rust/src/server/src/grpc/control.rs | 20 ++++++ rust/src/server/src/grpc/tests.rs | 62 +++++++++++++++++-- rust/src/server/src/lib.rs | 8 ++- 10 files changed, 129 insertions(+), 9 deletions(-) diff --git a/docs/usage/security.md b/docs/usage/security.md index a6c9867755..2b09ee5266 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -375,7 +375,7 @@ An attacker who can reach the gRPC port can: 1. **Run arbitrary inference** via the `Generate` and `GenerateStream` RPCs without any credentials 2. **Mutate engine state** by pausing generation, sleeping the engine, or initiating configured RL weight updates through the `Control` service 3. **Consume GPU and compute resources** by submitting unbounded generation requests -4. **Cause Denial of Service** by exploiting bugs in the gRPC interface that can crash vLLM. +4. **Stop a managed engine** through `Control.Shutdown`, or cause denial of service by exploiting bugs in the gRPC interface. ### Recommendations diff --git a/rust/proto/README.md b/rust/proto/README.md index e792cf79e2..d312e27aca 100644 --- a/rust/proto/README.md +++ b/rust/proto/README.md @@ -16,7 +16,7 @@ the schemas on pull requests, including on forks. ```toml [dependencies] -vllm-proto = "0.3" +vllm-proto = "0.4" ``` For example, import `vllm_proto::inference_client::InferenceClient` or diff --git a/rust/proto/control.proto b/rust/proto/control.proto index ef3218d35c..e42f958582 100644 --- a/rust/proto/control.proto +++ b/rust/proto/control.proto @@ -9,6 +9,10 @@ service Control { rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {} rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {} + // Request managed-engine shutdown using the configured shutdown timeout. + // Returns FAILED_PRECONDITION when this server does not manage the engine process. + rpc Shutdown (ShutdownRequest) returns (ShutdownResponse) {} + // Request lifecycle. rpc Abort (AbortRequest) returns (AbortResponse) {} @@ -86,6 +90,10 @@ message ModelInfo { string tool_call_parser = 25; } +message ShutdownRequest {} +// Acknowledges acceptance, not completion; shutdown may close the connection first. +message ShutdownResponse {} + message AbortRequest { repeated string request_ids = 1; } diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index f57d43a75f..1fab1c93d1 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -557,6 +557,8 @@ impl SharedRuntimeArgs { disable_log_stats: self.disable_log_stats, grpc_port: self.grpc_port, shutdown_timeout, + // The engine is launched and supervised by another process. + manages_engine: false, keep_alive_timeout, profiler, } @@ -572,6 +574,7 @@ impl SharedRuntimeArgs { engine_count: usize, local_input_address: Option, local_output_address: Option, + manages_engine: bool, ) -> Config { let ready_timeout = self.ready_timeout(); let shutdown_timeout = self.shutdown_timeout(); @@ -615,6 +618,7 @@ impl SharedRuntimeArgs { disable_log_stats: self.disable_log_stats, grpc_port: self.grpc_port, shutdown_timeout, + manages_engine, keep_alive_timeout, profiler, } @@ -793,6 +797,8 @@ impl ServeArgs { self.managed_engine.data_parallel_size, local_input_address, local_output_address, + // `--data-parallel-size-local 0` runs the frontend without a local engine. + self.managed_engine.data_parallel_size_local != Some(0), ) } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index dac00ac983..0802d2062a 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -1889,6 +1889,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { disable_log_stats: false, grpc_port: None, shutdown_timeout: 0ns, + manages_engine: true, keep_alive_timeout: 5s, profiler: None, } @@ -1983,6 +1984,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { disable_log_stats: false, grpc_port: None, shutdown_timeout: 0ns, + manages_engine: true, keep_alive_timeout: 5s, profiler: None, } @@ -2011,6 +2013,29 @@ fn frontend_args_reject_legacy_handshake_flags() { assert!(error.to_string().contains("--handshake-address")); } +#[test] +fn serve_frontend_config_does_not_manage_engine_without_local_engines() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--data-parallel-address", + "10.99.48.128", + "--data-parallel-size", + "2", + "--data-parallel-size-local", + "0", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string()); + + assert!(!config.manages_engine); +} + #[test] fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() { let cli = Cli::try_parse_from([ @@ -2098,6 +2123,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present disable_log_stats: false, grpc_port: None, shutdown_timeout: 0ns, + manages_engine: false, keep_alive_timeout: 5s, profiler: None, } diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 3de227babd..cb5fed4437 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -87,6 +87,7 @@ async fn main() -> Result<()> { disable_log_stats: false, grpc_port: None, shutdown_timeout: Duration::ZERO, + manages_engine: false, keep_alive_timeout: Duration::from_secs(5), profiler: None, }; diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index a9a77a9a9e..b307f8b3ac 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -277,6 +277,9 @@ pub struct Config { pub grpc_port: Option, /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown. pub shutdown_timeout: Duration, + /// Whether the caller manages the engine process and shuts it down when + /// the shutdown token is cancelled. Enables the gRPC `Control.Shutdown` RPC. + pub manages_engine: bool, /// Maximum idle time on a keep-alive HTTP connection before the server /// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s). pub keep_alive_timeout: Duration, diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index 680ebdfbda..d1b5ac4fe0 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use serde_json::Value as JsonValue; use thiserror_ext::AsReport as _; use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; use tonic::{Code, Request, Response, Status}; use vllm_engine_core_client::EngineCoreClient; use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse; @@ -23,6 +24,7 @@ pub(crate) type ControlGrpcService = ControlServer; pub struct ControlServiceImpl { state: Arc, rl_lock: Mutex<()>, + engine_shutdown: Option, } impl ControlServiceImpl { @@ -30,9 +32,15 @@ impl ControlServiceImpl { Self { state, rl_lock: Mutex::new(()), + engine_shutdown: None, } } + pub fn with_engine_shutdown(mut self, shutdown: Option) -> Self { + self.engine_shutdown = shutdown; + self + } + fn ready(&self) -> &EngineCoreReadyResponse { self.state.engine_core_client().ready_response() } @@ -179,6 +187,18 @@ fn list_loras_status(error: LoraDisabledError) -> Status { #[tonic::async_trait] impl pb::control_server::Control for ControlServiceImpl { + async fn shutdown( + &self, + _request: Request, + ) -> Result, Status> { + let shutdown = self.engine_shutdown.as_ref().ok_or_else(|| { + Status::failed_precondition("this frontend does not manage the engine process") + })?; + tracing::info!("engine shutdown requested via gRPC"); + shutdown.cancel(); + Ok(Response::new(pb::ShutdownResponse {})) + } + async fn get_server_info( &self, _request: Request, diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 21b240c4b1..9f01886062 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -366,6 +366,7 @@ where engine_id, default_ready_response(), backend, + None, move |dealer, push| { boxed_test_future(async move { let add = recv_engine_message(dealer).await; @@ -387,6 +388,7 @@ async fn setup_grpc_service_with_engine_script( engine_id: impl Into, ready: EngineCoreReadyResponse, backend: Arc, + engine_shutdown: Option, script: F, ) -> ( InferenceServer, @@ -425,7 +427,7 @@ where ( InferenceServer::new(InferenceServiceImpl::new(state.clone())) .max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES), - ControlServer::new(ControlServiceImpl::new(state)), + ControlServer::new(ControlServiceImpl::new(state).with_engine_shutdown(engine_shutdown)), engine_health, engine_task, ) @@ -1920,6 +1922,7 @@ async fn control_reports_server_and_model_info() { b"engine-grpc-info".to_vec(), ready, Arc::new(FakeTextBackend), + None, |_, _| boxed_test_future(async {}), ) .await; @@ -1987,6 +1990,7 @@ async fn control_lora_lifecycle_selects_adapter_for_generation() { b"engine-grpc-lora".to_vec(), ready, Arc::new(FakeTextBackend), + None, |dealer, push| { boxed_test_future(async move { reply_utility_bool(dealer, push, "add_lora", true).await; @@ -2111,6 +2115,7 @@ async fn control_list_loras_requires_lora_enabled_engine() { b"engine-grpc-lora-disabled".to_vec(), default_ready_response(), Arc::new(FakeTextBackend), + None, |_, _| boxed_test_future(async move {}), ) .await; @@ -2143,6 +2148,7 @@ async fn control_forwards_weight_update_without_pause_guard() { b"engine-grpc-rl".to_vec(), ready, Arc::new(FakeTextBackend), + None, |dealer, push| { boxed_test_future(async move { let frames = recv_engine_message(dealer).await; @@ -2392,9 +2398,9 @@ async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy() #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn grpc_health_watch_closes_on_graceful_shutdown() { +async fn grpc_shutdown_requires_managed_engine() { let (inference_service, control_service, engine_health, _engine_task) = setup_grpc_service( - b"engine-grpc-health-shutdown", + b"engine-grpc-external-shutdown", default_stream_output_specs(), ) .await; @@ -2406,6 +2412,47 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() { shutdown.clone(), ) .await; + let mut control_client = ControlClient::new(channel.clone()); + + let status = control_client + .shutdown(pb::ShutdownRequest {}) + .await + .expect_err("external engine shutdown must be rejected"); + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert!(!shutdown.is_cancelled()); + + let health = HealthClient::new(channel) + .check(HealthCheckRequest { + service: "vllm.Inference".to_string(), + }) + .await + .expect("server remains healthy after rejected shutdown") + .into_inner(); + assert_eq!(health.status, HealthServingStatus::Serving as i32); + server_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn grpc_shutdown_closes_health_watch_and_server() { + let shutdown = tokio_util::sync::CancellationToken::new(); + let (inference_service, control_service, engine_health, _engine_task) = + setup_grpc_service_with_engine_script( + b"engine-grpc-health-shutdown", + default_ready_response(), + Arc::new(FakeTextBackend), + Some(shutdown.clone()), + |_, _| boxed_test_future(async {}), + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + shutdown.child_token(), + ) + .await; + let mut control_client = ControlClient::new(channel.clone()); let mut health_client = HealthClient::new(channel); let mut stream = health_client .watch(HealthCheckRequest { @@ -2426,7 +2473,14 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() { "unexpected initial health status for vllm.Inference" ); - shutdown.cancel(); + control_client + .shutdown(pb::ShutdownRequest {}) + .await + .expect("accept managed engine shutdown"); + assert!( + shutdown.is_cancelled(), + "engine owner must receive shutdown" + ); let update = tokio::time::timeout(Duration::from_secs(2), stream.message()) .await diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index d5f9e1ea55..6a36491fd9 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -247,9 +247,11 @@ where let engine_health = state.engine_core_client().subscribe_health(); health_reporter.set_serving::().await; health_reporter.set_serving::().await; - let control_service = - grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone())) - .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES); + let control_service = grpc::ControlGrpcService::new( + grpc::ControlServiceImpl::new(state.clone()) + .with_engine_shutdown(config.manages_engine.then(|| shutdown.clone())), + ) + .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES); let inference_service = grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone())) .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);