[Feature][Rust Frontend] Add Shutdown control RPC (#59316)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Julien Darve <jdarve@NVIDIA.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
JulienDarve
2026-09-30 12:49:25 +00:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent b22494cc0c
commit f28a508162
10 changed files with 129 additions and 9 deletions
+1 -1
View File
@@ -375,7 +375,7 @@ An attacker who can reach the gRPC port can:
1. **Run arbitrary inference** via the `Generate` and `GenerateStream` RPCs without any credentials
2. **Mutate engine state** by pausing generation, sleeping the engine, or initiating configured RL weight updates through the `Control` service
3. **Consume GPU and compute resources** by submitting unbounded generation requests
4. **Cause Denial of Service** by exploiting bugs in the gRPC interface that can crash vLLM.
4. **Stop a managed engine** through `Control.Shutdown`, or cause denial of service by exploiting bugs in the gRPC interface.
### Recommendations
+1 -1
View File
@@ -16,7 +16,7 @@ the schemas on pull requests, including on forks.
```toml
[dependencies]
vllm-proto = "0.3"
vllm-proto = "0.4"
```
For example, import `vllm_proto::inference_client::InferenceClient` or
+8
View File
@@ -9,6 +9,10 @@ service Control {
rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {}
rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {}
// Request managed-engine shutdown using the configured shutdown timeout.
// Returns FAILED_PRECONDITION when this server does not manage the engine process.
rpc Shutdown (ShutdownRequest) returns (ShutdownResponse) {}
// Request lifecycle.
rpc Abort (AbortRequest) returns (AbortResponse) {}
@@ -86,6 +90,10 @@ message ModelInfo {
string tool_call_parser = 25;
}
message ShutdownRequest {}
// Acknowledges acceptance, not completion; shutdown may close the connection first.
message ShutdownResponse {}
message AbortRequest {
repeated string request_ids = 1;
}
+6
View File
@@ -557,6 +557,8 @@ impl SharedRuntimeArgs {
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
// The engine is launched and supervised by another process.
manages_engine: false,
keep_alive_timeout,
profiler,
}
@@ -572,6 +574,7 @@ impl SharedRuntimeArgs {
engine_count: usize,
local_input_address: Option<String>,
local_output_address: Option<String>,
manages_engine: bool,
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
@@ -615,6 +618,7 @@ impl SharedRuntimeArgs {
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
manages_engine,
keep_alive_timeout,
profiler,
}
@@ -793,6 +797,8 @@ impl ServeArgs {
self.managed_engine.data_parallel_size,
local_input_address,
local_output_address,
// `--data-parallel-size-local 0` runs the frontend without a local engine.
self.managed_engine.data_parallel_size_local != Some(0),
)
}
+26
View File
@@ -1889,6 +1889,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: true,
keep_alive_timeout: 5s,
profiler: None,
}
@@ -1983,6 +1984,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: true,
keep_alive_timeout: 5s,
profiler: None,
}
@@ -2011,6 +2013,29 @@ fn frontend_args_reject_legacy_handshake_flags() {
assert!(error.to_string().contains("--handshake-address"));
}
#[test]
fn serve_frontend_config_does_not_manage_engine_without_local_engines() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--data-parallel-address",
"10.99.48.128",
"--data-parallel-size",
"2",
"--data-parallel-size-local",
"0",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
assert!(!config.manages_engine);
}
#[test]
fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() {
let cli = Cli::try_parse_from([
@@ -2098,6 +2123,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: false,
keep_alive_timeout: 5s,
profiler: None,
}
@@ -87,6 +87,7 @@ async fn main() -> Result<()> {
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: Duration::ZERO,
manages_engine: false,
keep_alive_timeout: Duration::from_secs(5),
profiler: None,
};
+3
View File
@@ -277,6 +277,9 @@ pub struct Config {
pub grpc_port: Option<u16>,
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
pub shutdown_timeout: Duration,
/// Whether the caller manages the engine process and shuts it down when
/// the shutdown token is cancelled. Enables the gRPC `Control.Shutdown` RPC.
pub manages_engine: bool,
/// Maximum idle time on a keep-alive HTTP connection before the server
/// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
pub keep_alive_timeout: Duration,
+20
View File
@@ -6,6 +6,7 @@ use std::sync::Arc;
use serde_json::Value as JsonValue;
use thiserror_ext::AsReport as _;
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, Response, Status};
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
@@ -23,6 +24,7 @@ pub(crate) type ControlGrpcService = ControlServer<ControlServiceImpl>;
pub struct ControlServiceImpl {
state: Arc<AppState>,
rl_lock: Mutex<()>,
engine_shutdown: Option<CancellationToken>,
}
impl ControlServiceImpl {
@@ -30,9 +32,15 @@ impl ControlServiceImpl {
Self {
state,
rl_lock: Mutex::new(()),
engine_shutdown: None,
}
}
pub fn with_engine_shutdown(mut self, shutdown: Option<CancellationToken>) -> Self {
self.engine_shutdown = shutdown;
self
}
fn ready(&self) -> &EngineCoreReadyResponse {
self.state.engine_core_client().ready_response()
}
@@ -179,6 +187,18 @@ fn list_loras_status(error: LoraDisabledError) -> Status {
#[tonic::async_trait]
impl pb::control_server::Control for ControlServiceImpl {
async fn shutdown(
&self,
_request: Request<pb::ShutdownRequest>,
) -> Result<Response<pb::ShutdownResponse>, Status> {
let shutdown = self.engine_shutdown.as_ref().ok_or_else(|| {
Status::failed_precondition("this frontend does not manage the engine process")
})?;
tracing::info!("engine shutdown requested via gRPC");
shutdown.cancel();
Ok(Response::new(pb::ShutdownResponse {}))
}
async fn get_server_info(
&self,
_request: Request<pb::GetServerInfoRequest>,
+58 -4
View File
@@ -366,6 +366,7 @@ where
engine_id,
default_ready_response(),
backend,
None,
move |dealer, push| {
boxed_test_future(async move {
let add = recv_engine_message(dealer).await;
@@ -387,6 +388,7 @@ async fn setup_grpc_service_with_engine_script<F>(
engine_id: impl Into<EngineId>,
ready: EngineCoreReadyResponse,
backend: Arc<dyn ChatTextBackend>,
engine_shutdown: Option<tokio_util::sync::CancellationToken>,
script: F,
) -> (
InferenceServer<InferenceServiceImpl>,
@@ -425,7 +427,7 @@ where
(
InferenceServer::new(InferenceServiceImpl::new(state.clone()))
.max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES),
ControlServer::new(ControlServiceImpl::new(state)),
ControlServer::new(ControlServiceImpl::new(state).with_engine_shutdown(engine_shutdown)),
engine_health,
engine_task,
)
@@ -1920,6 +1922,7 @@ async fn control_reports_server_and_model_info() {
b"engine-grpc-info".to_vec(),
ready,
Arc::new(FakeTextBackend),
None,
|_, _| boxed_test_future(async {}),
)
.await;
@@ -1987,6 +1990,7 @@ async fn control_lora_lifecycle_selects_adapter_for_generation() {
b"engine-grpc-lora".to_vec(),
ready,
Arc::new(FakeTextBackend),
None,
|dealer, push| {
boxed_test_future(async move {
reply_utility_bool(dealer, push, "add_lora", true).await;
@@ -2111,6 +2115,7 @@ async fn control_list_loras_requires_lora_enabled_engine() {
b"engine-grpc-lora-disabled".to_vec(),
default_ready_response(),
Arc::new(FakeTextBackend),
None,
|_, _| boxed_test_future(async move {}),
)
.await;
@@ -2143,6 +2148,7 @@ async fn control_forwards_weight_update_without_pause_guard() {
b"engine-grpc-rl".to_vec(),
ready,
Arc::new(FakeTextBackend),
None,
|dealer, push| {
boxed_test_future(async move {
let frames = recv_engine_message(dealer).await;
@@ -2392,9 +2398,9 @@ async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy()
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn grpc_health_watch_closes_on_graceful_shutdown() {
async fn grpc_shutdown_requires_managed_engine() {
let (inference_service, control_service, engine_health, _engine_task) = setup_grpc_service(
b"engine-grpc-health-shutdown",
b"engine-grpc-external-shutdown",
default_stream_output_specs(),
)
.await;
@@ -2406,6 +2412,47 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() {
shutdown.clone(),
)
.await;
let mut control_client = ControlClient::new(channel.clone());
let status = control_client
.shutdown(pb::ShutdownRequest {})
.await
.expect_err("external engine shutdown must be rejected");
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
assert!(!shutdown.is_cancelled());
let health = HealthClient::new(channel)
.check(HealthCheckRequest {
service: "vllm.Inference".to_string(),
})
.await
.expect("server remains healthy after rejected shutdown")
.into_inner();
assert_eq!(health.status, HealthServingStatus::Serving as i32);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn grpc_shutdown_closes_health_watch_and_server() {
let shutdown = tokio_util::sync::CancellationToken::new();
let (inference_service, control_service, engine_health, _engine_task) =
setup_grpc_service_with_engine_script(
b"engine-grpc-health-shutdown",
default_ready_response(),
Arc::new(FakeTextBackend),
Some(shutdown.clone()),
|_, _| boxed_test_future(async {}),
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
shutdown.child_token(),
)
.await;
let mut control_client = ControlClient::new(channel.clone());
let mut health_client = HealthClient::new(channel);
let mut stream = health_client
.watch(HealthCheckRequest {
@@ -2426,7 +2473,14 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() {
"unexpected initial health status for vllm.Inference"
);
shutdown.cancel();
control_client
.shutdown(pb::ShutdownRequest {})
.await
.expect("accept managed engine shutdown");
assert!(
shutdown.is_cancelled(),
"engine owner must receive shutdown"
);
let update = tokio::time::timeout(Duration::from_secs(2), stream.message())
.await
+5 -3
View File
@@ -247,9 +247,11 @@ where
let engine_health = state.engine_core_client().subscribe_health();
health_reporter.set_serving::<grpc::InferenceGrpcService>().await;
health_reporter.set_serving::<grpc::ControlGrpcService>().await;
let control_service =
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()))
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
let control_service = grpc::ControlGrpcService::new(
grpc::ControlServiceImpl::new(state.clone())
.with_engine_shutdown(config.manages_engine.then(|| shutdown.clone())),
)
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
let inference_service =
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()))
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);