[Feature][Rust Frontend] Add Shutdown control RPC (#59316)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Julien Darve <jdarve@NVIDIA.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
b22494cc0c
commit
f28a508162
@@ -375,7 +375,7 @@ An attacker who can reach the gRPC port can:
|
||||
1. **Run arbitrary inference** via the `Generate` and `GenerateStream` RPCs without any credentials
|
||||
2. **Mutate engine state** by pausing generation, sleeping the engine, or initiating configured RL weight updates through the `Control` service
|
||||
3. **Consume GPU and compute resources** by submitting unbounded generation requests
|
||||
4. **Cause Denial of Service** by exploiting bugs in the gRPC interface that can crash vLLM.
|
||||
4. **Stop a managed engine** through `Control.Shutdown`, or cause denial of service by exploiting bugs in the gRPC interface.
|
||||
|
||||
### Recommendations
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ the schemas on pull requests, including on forks.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
vllm-proto = "0.3"
|
||||
vllm-proto = "0.4"
|
||||
```
|
||||
|
||||
For example, import `vllm_proto::inference_client::InferenceClient` or
|
||||
|
||||
@@ -9,6 +9,10 @@ service Control {
|
||||
rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {}
|
||||
rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {}
|
||||
|
||||
// Request managed-engine shutdown using the configured shutdown timeout.
|
||||
// Returns FAILED_PRECONDITION when this server does not manage the engine process.
|
||||
rpc Shutdown (ShutdownRequest) returns (ShutdownResponse) {}
|
||||
|
||||
// Request lifecycle.
|
||||
rpc Abort (AbortRequest) returns (AbortResponse) {}
|
||||
|
||||
@@ -86,6 +90,10 @@ message ModelInfo {
|
||||
string tool_call_parser = 25;
|
||||
}
|
||||
|
||||
message ShutdownRequest {}
|
||||
// Acknowledges acceptance, not completion; shutdown may close the connection first.
|
||||
message ShutdownResponse {}
|
||||
|
||||
message AbortRequest {
|
||||
repeated string request_ids = 1;
|
||||
}
|
||||
|
||||
@@ -557,6 +557,8 @@ impl SharedRuntimeArgs {
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
// The engine is launched and supervised by another process.
|
||||
manages_engine: false,
|
||||
keep_alive_timeout,
|
||||
profiler,
|
||||
}
|
||||
@@ -572,6 +574,7 @@ impl SharedRuntimeArgs {
|
||||
engine_count: usize,
|
||||
local_input_address: Option<String>,
|
||||
local_output_address: Option<String>,
|
||||
manages_engine: bool,
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
@@ -615,6 +618,7 @@ impl SharedRuntimeArgs {
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
manages_engine,
|
||||
keep_alive_timeout,
|
||||
profiler,
|
||||
}
|
||||
@@ -793,6 +797,8 @@ impl ServeArgs {
|
||||
self.managed_engine.data_parallel_size,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
// `--data-parallel-size-local 0` runs the frontend without a local engine.
|
||||
self.managed_engine.data_parallel_size_local != Some(0),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1889,6 +1889,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
manages_engine: true,
|
||||
keep_alive_timeout: 5s,
|
||||
profiler: None,
|
||||
}
|
||||
@@ -1983,6 +1984,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
manages_engine: true,
|
||||
keep_alive_timeout: 5s,
|
||||
profiler: None,
|
||||
}
|
||||
@@ -2011,6 +2013,29 @@ fn frontend_args_reject_legacy_handshake_flags() {
|
||||
assert!(error.to_string().contains("--handshake-address"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_does_not_manage_engine_without_local_engines() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--data-parallel-address",
|
||||
"10.99.48.128",
|
||||
"--data-parallel-size",
|
||||
"2",
|
||||
"--data-parallel-size-local",
|
||||
"0",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
|
||||
|
||||
assert!(!config.manages_engine);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() {
|
||||
let cli = Cli::try_parse_from([
|
||||
@@ -2098,6 +2123,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
manages_engine: false,
|
||||
keep_alive_timeout: 5s,
|
||||
profiler: None,
|
||||
}
|
||||
|
||||
@@ -87,6 +87,7 @@ async fn main() -> Result<()> {
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: Duration::ZERO,
|
||||
manages_engine: false,
|
||||
keep_alive_timeout: Duration::from_secs(5),
|
||||
profiler: None,
|
||||
};
|
||||
|
||||
@@ -277,6 +277,9 @@ pub struct Config {
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
|
||||
pub shutdown_timeout: Duration,
|
||||
/// Whether the caller manages the engine process and shuts it down when
|
||||
/// the shutdown token is cancelled. Enables the gRPC `Control.Shutdown` RPC.
|
||||
pub manages_engine: bool,
|
||||
/// Maximum idle time on a keep-alive HTTP connection before the server
|
||||
/// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
|
||||
pub keep_alive_timeout: Duration,
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::sync::Arc;
|
||||
use serde_json::Value as JsonValue;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, Response, Status};
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
|
||||
@@ -23,6 +24,7 @@ pub(crate) type ControlGrpcService = ControlServer<ControlServiceImpl>;
|
||||
pub struct ControlServiceImpl {
|
||||
state: Arc<AppState>,
|
||||
rl_lock: Mutex<()>,
|
||||
engine_shutdown: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
impl ControlServiceImpl {
|
||||
@@ -30,9 +32,15 @@ impl ControlServiceImpl {
|
||||
Self {
|
||||
state,
|
||||
rl_lock: Mutex::new(()),
|
||||
engine_shutdown: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_engine_shutdown(mut self, shutdown: Option<CancellationToken>) -> Self {
|
||||
self.engine_shutdown = shutdown;
|
||||
self
|
||||
}
|
||||
|
||||
fn ready(&self) -> &EngineCoreReadyResponse {
|
||||
self.state.engine_core_client().ready_response()
|
||||
}
|
||||
@@ -179,6 +187,18 @@ fn list_loras_status(error: LoraDisabledError) -> Status {
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl pb::control_server::Control for ControlServiceImpl {
|
||||
async fn shutdown(
|
||||
&self,
|
||||
_request: Request<pb::ShutdownRequest>,
|
||||
) -> Result<Response<pb::ShutdownResponse>, Status> {
|
||||
let shutdown = self.engine_shutdown.as_ref().ok_or_else(|| {
|
||||
Status::failed_precondition("this frontend does not manage the engine process")
|
||||
})?;
|
||||
tracing::info!("engine shutdown requested via gRPC");
|
||||
shutdown.cancel();
|
||||
Ok(Response::new(pb::ShutdownResponse {}))
|
||||
}
|
||||
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
_request: Request<pb::GetServerInfoRequest>,
|
||||
|
||||
@@ -366,6 +366,7 @@ where
|
||||
engine_id,
|
||||
default_ready_response(),
|
||||
backend,
|
||||
None,
|
||||
move |dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
@@ -387,6 +388,7 @@ async fn setup_grpc_service_with_engine_script<F>(
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready: EngineCoreReadyResponse,
|
||||
backend: Arc<dyn ChatTextBackend>,
|
||||
engine_shutdown: Option<tokio_util::sync::CancellationToken>,
|
||||
script: F,
|
||||
) -> (
|
||||
InferenceServer<InferenceServiceImpl>,
|
||||
@@ -425,7 +427,7 @@ where
|
||||
(
|
||||
InferenceServer::new(InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES),
|
||||
ControlServer::new(ControlServiceImpl::new(state)),
|
||||
ControlServer::new(ControlServiceImpl::new(state).with_engine_shutdown(engine_shutdown)),
|
||||
engine_health,
|
||||
engine_task,
|
||||
)
|
||||
@@ -1920,6 +1922,7 @@ async fn control_reports_server_and_model_info() {
|
||||
b"engine-grpc-info".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
@@ -1987,6 +1990,7 @@ async fn control_lora_lifecycle_selects_adapter_for_generation() {
|
||||
b"engine-grpc-lora".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
reply_utility_bool(dealer, push, "add_lora", true).await;
|
||||
@@ -2111,6 +2115,7 @@ async fn control_list_loras_requires_lora_enabled_engine() {
|
||||
b"engine-grpc-lora-disabled".to_vec(),
|
||||
default_ready_response(),
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|_, _| boxed_test_future(async move {}),
|
||||
)
|
||||
.await;
|
||||
@@ -2143,6 +2148,7 @@ async fn control_forwards_weight_update_without_pause_guard() {
|
||||
b"engine-grpc-rl".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
None,
|
||||
|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let frames = recv_engine_message(dealer).await;
|
||||
@@ -2392,9 +2398,9 @@ async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy()
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn grpc_health_watch_closes_on_graceful_shutdown() {
|
||||
async fn grpc_shutdown_requires_managed_engine() {
|
||||
let (inference_service, control_service, engine_health, _engine_task) = setup_grpc_service(
|
||||
b"engine-grpc-health-shutdown",
|
||||
b"engine-grpc-external-shutdown",
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
@@ -2406,6 +2412,47 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() {
|
||||
shutdown.clone(),
|
||||
)
|
||||
.await;
|
||||
let mut control_client = ControlClient::new(channel.clone());
|
||||
|
||||
let status = control_client
|
||||
.shutdown(pb::ShutdownRequest {})
|
||||
.await
|
||||
.expect_err("external engine shutdown must be rejected");
|
||||
assert_eq!(status.code(), tonic::Code::FailedPrecondition);
|
||||
assert!(!shutdown.is_cancelled());
|
||||
|
||||
let health = HealthClient::new(channel)
|
||||
.check(HealthCheckRequest {
|
||||
service: "vllm.Inference".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("server remains healthy after rejected shutdown")
|
||||
.into_inner();
|
||||
assert_eq!(health.status, HealthServingStatus::Serving as i32);
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn grpc_shutdown_closes_health_watch_and_server() {
|
||||
let shutdown = tokio_util::sync::CancellationToken::new();
|
||||
let (inference_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service_with_engine_script(
|
||||
b"engine-grpc-health-shutdown",
|
||||
default_ready_response(),
|
||||
Arc::new(FakeTextBackend),
|
||||
Some(shutdown.clone()),
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
shutdown.child_token(),
|
||||
)
|
||||
.await;
|
||||
let mut control_client = ControlClient::new(channel.clone());
|
||||
let mut health_client = HealthClient::new(channel);
|
||||
let mut stream = health_client
|
||||
.watch(HealthCheckRequest {
|
||||
@@ -2426,7 +2473,14 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() {
|
||||
"unexpected initial health status for vllm.Inference"
|
||||
);
|
||||
|
||||
shutdown.cancel();
|
||||
control_client
|
||||
.shutdown(pb::ShutdownRequest {})
|
||||
.await
|
||||
.expect("accept managed engine shutdown");
|
||||
assert!(
|
||||
shutdown.is_cancelled(),
|
||||
"engine owner must receive shutdown"
|
||||
);
|
||||
|
||||
let update = tokio::time::timeout(Duration::from_secs(2), stream.message())
|
||||
.await
|
||||
|
||||
@@ -247,9 +247,11 @@ where
|
||||
let engine_health = state.engine_core_client().subscribe_health();
|
||||
health_reporter.set_serving::<grpc::InferenceGrpcService>().await;
|
||||
health_reporter.set_serving::<grpc::ControlGrpcService>().await;
|
||||
let control_service =
|
||||
grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
let control_service = grpc::ControlGrpcService::new(
|
||||
grpc::ControlServiceImpl::new(state.clone())
|
||||
.with_engine_shutdown(config.manages_engine.then(|| shutdown.clone())),
|
||||
)
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
let inference_service =
|
||||
grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone()))
|
||||
.max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES);
|
||||
|
||||
Reference in New Issue
Block a user