[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Alec
2026-10-02 08:14:37 +00:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent b558f160a2
commit 9d2a6f52b2
11 changed files with 320 additions and 64 deletions
+24 -7
View File
@@ -268,11 +268,6 @@ pub struct SharedRuntimeArgs {
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
#[serde(default)]
pub max_logprobs: Option<i32>,
/// TCP port for the gRPC Inference service. When not set, no gRPC server is
/// started.
#[arg(long)]
#[serde(default)]
pub grpc_port: Option<u16>,
/// Maximum time to wait for active requests to drain during shutdown.
#[arg(long, default_value_t = 0)]
#[serde(default)]
@@ -505,6 +500,7 @@ impl SharedRuntimeArgs {
fn into_bootstrapped_config(
self,
listen_fd: i32,
grpc_listen_fd: Option<i32>,
input_address: String,
output_address: String,
coordinator_address: Option<String>,
@@ -539,6 +535,7 @@ impl SharedRuntimeArgs {
generation_config: self.generation_config,
served_model_name: self.served_model_name,
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
grpc_listener_mode: grpc_listen_fd.map(|fd| HttpListenerMode::InheritedFd { fd }),
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
@@ -555,7 +552,6 @@ impl SharedRuntimeArgs {
tls,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
// The engine is launched and supervised by another process.
manages_engine: false,
@@ -569,6 +565,7 @@ impl SharedRuntimeArgs {
fn into_managed_config(
self,
listener_mode: HttpListenerMode,
grpc_listener_mode: Option<HttpListenerMode>,
handshake_address: String,
advertised_host: String,
engine_count: usize,
@@ -600,6 +597,7 @@ impl SharedRuntimeArgs {
generation_config: self.generation_config,
served_model_name: self.served_model_name,
listener_mode,
grpc_listener_mode,
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
@@ -616,7 +614,6 @@ impl SharedRuntimeArgs {
tls,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
manages_engine,
keep_alive_timeout,
@@ -696,6 +693,10 @@ pub struct FrontendArgs {
/// supervisor.
#[arg(long)]
pub listen_fd: i32,
/// Inherited gRPC listening socket file descriptor passed by the Python
/// supervisor. When not set, no gRPC server is started.
#[arg(long)]
pub grpc_listen_fd: Option<i32>,
/// Frontend input ROUTER socket address that the Python engines will
/// connect to.
#[arg(long)]
@@ -731,6 +732,7 @@ impl FrontendArgs {
let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
self.runtime.into_bootstrapped_config(
self.listen_fd,
self.grpc_listen_fd,
self.input_address,
self.output_address,
self.coordinator_address,
@@ -760,6 +762,10 @@ pub struct ServeArgs {
/// Unix domain socket path. If set, host and port arguments are ignored.
#[arg(long)]
pub uds: Option<String>,
/// TCP port for the gRPC Inference and Control services. When not set, no
/// gRPC server is started.
#[arg(long)]
pub grpc_port: Option<u16>,
/// Flag to print debug information about CLI argument parsing and exit.
#[educe(Debug(ignore))]
@@ -789,9 +795,20 @@ impl ServeArgs {
port: self.port,
},
};
// gRPC follows the HTTP TCP host. With a Unix socket it defaults to IPv4
// loopback rather than all interfaces, so the side-car is never
// accidentally network-exposed.
let grpc_listener_mode = self.grpc_port.map(|port| HttpListenerMode::BindTcp {
host: match self.uds {
Some(_) => "127.0.0.1".to_string(),
None => self.host.clone(),
},
port,
});
self.runtime.clone().into_managed_config(
listener_mode,
grpc_listener_mode,
handshake_address,
self.managed_engine.handshake_host.clone(),
self.managed_engine.data_parallel_size,
+71 -6
View File
@@ -174,6 +174,7 @@ fn serve_args_forward_python_flags_with_separator() {
host: "127.0.0.1",
port: 8000,
uds: None,
grpc_port: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
revision: None,
@@ -188,7 +189,6 @@ fn serve_args_forward_python_flags_with_separator() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1007,6 +1007,7 @@ fn frontend_args_accept_json() {
command: Frontend(
FrontendArgs {
listen_fd: 3,
grpc_listen_fd: None,
input_address: "ipc:///tmp/input.sock",
output_address: "ipc:///tmp/output.sock",
coordinator_address: Some(
@@ -1029,7 +1030,6 @@ fn frontend_args_accept_json() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1687,6 +1687,7 @@ fn serve_args_accept_handshake_aliases() {
host: "127.0.0.1",
port: 8000,
uds: None,
grpc_port: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
revision: None,
@@ -1701,7 +1702,6 @@ fn serve_args_accept_handshake_aliases() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1854,6 +1854,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
host: "127.0.0.1",
port: 8000,
},
grpc_listener_mode: None,
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
@@ -1887,7 +1888,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
tls: None,
api_keys: [],
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: true,
keep_alive_timeout: 5s,
@@ -1949,6 +1949,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
host: "127.0.0.1",
port: 8000,
},
grpc_listener_mode: None,
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
@@ -1982,7 +1983,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
tls: None,
api_keys: [],
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: true,
keep_alive_timeout: 5s,
@@ -2088,6 +2088,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
listener_mode: InheritedFd {
fd: 3,
},
grpc_listener_mode: None,
tool_call_parser: None,
reasoning_parser: None,
tool_strict_level: Auto,
@@ -2121,7 +2122,6 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
tls: None,
api_keys: [],
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: false,
keep_alive_timeout: 5s,
@@ -2139,6 +2139,8 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
"Qwen/Qwen3-0.6B",
"--uds",
"/tmp/vllm.sock",
"--grpc-port",
"50051",
])
.unwrap();
@@ -2153,6 +2155,69 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
path: "/tmp/vllm.sock".to_string(),
}
);
assert_eq!(
config.grpc_listener_mode,
Some(HttpListenerMode::BindTcp {
host: "127.0.0.1".to_string(),
port: 50051,
})
);
}
#[test]
fn serve_frontend_config_binds_grpc_on_http_host() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--host",
"0.0.0.0",
"--grpc-port",
"50051",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
assert_eq!(
config.grpc_listener_mode,
Some(HttpListenerMode::BindTcp {
host: "0.0.0.0".to_string(),
port: 50051,
})
);
}
#[test]
fn frontend_config_adopts_inherited_grpc_listener() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--grpc-listen-fd",
"4",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","grpc_port":50051}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
let config = args.into_config();
assert_eq!(
config.grpc_listener_mode,
Some(HttpListenerMode::InheritedFd { fd: 4 })
);
}
#[test]
@@ -69,6 +69,7 @@ async fn main() -> Result<()> {
host: "127.0.0.1".to_string(),
port,
},
grpc_listener_mode: None,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
@@ -85,7 +86,6 @@ async fn main() -> Result<()> {
tls: None,
api_keys: Vec::new(),
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: Duration::ZERO,
manages_engine: false,
keep_alive_timeout: Duration::from_secs(5),
+3 -4
View File
@@ -23,7 +23,7 @@ use vllm_text::backend::hf::HfOverrides;
/// when keep-alive is disabled (`0`).
pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
/// How the HTTP server obtains its listening socket.
/// How the HTTP or gRPC server obtains its listening socket.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum HttpListenerMode {
/// Bind a fresh TCP listener on the given host/port.
@@ -229,6 +229,8 @@ pub struct Config {
pub served_model_name: Vec<String>,
/// HTTP listener setup.
pub listener_mode: HttpListenerMode,
/// gRPC listener setup. When `None`, no gRPC server is started.
pub grpc_listener_mode: Option<HttpListenerMode>,
/// Tool-call parser selection.
pub tool_call_parser: ParserSelection,
/// Reasoning parser selection.
@@ -272,9 +274,6 @@ pub struct Config {
/// engine-reported scheduler stats and request lifecycle events are not
/// exported.
pub disable_log_stats: bool,
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
/// started.
pub grpc_port: Option<u16>,
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
pub shutdown_timeout: Duration,
/// Whether the caller manages the engine process and shuts it down when
+4 -37
View File
@@ -37,7 +37,6 @@ use hyper_util::rt::{TokioIo, TokioTimer};
use hyper_util::server::graceful::GracefulShutdown;
use hyper_util::service::TowerToHyperService;
pub use render::{RenderConfig, serve_render};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
use tokio_util::sync::CancellationToken;
use tonic::transport::Server as TonicServer;
@@ -74,17 +73,6 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve
}
}
/// Choose the gRPC listener host. It follows the HTTP TCP host when there is
/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback
/// rather than all interfaces, so the side-car is never accidentally
/// network-exposed.
fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
match listener_mode {
HttpListenerMode::BindTcp { host, .. } => host.as_str(),
HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1",
}
}
/// Build the shared application state for one configured model and one engine
/// client, including any static LoRA adapters from `--lora-modules`.
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
@@ -229,13 +217,11 @@ where
// Optionally bind the gRPC Inference server on a separate port. Bind
// synchronously here so bind errors (port in use, permission denied, ...)
// surface before serving rather than being deferred until shutdown.
let grpc_setup = if let Some(grpc_port) = config.grpc_port {
let grpc_host = grpc_bind_host(&config.listener_mode);
let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
let grpc_setup = if let Some(grpc_listener_mode) = &config.grpc_listener_mode {
let grpc_listener = Listener::bind(grpc_listener_mode)
.await
.with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
let addr = grpc_listener.local_addr()?;
let grpc_listener = Listener::Tcp(grpc_listener);
.with_context(|| format!("failed to bind gRPC listener for {grpc_listener_mode:?}"))?;
let addr = grpc_listener.local_addr_display()?;
// gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2.
let grpc_tls = config
.tls
@@ -479,23 +465,4 @@ mod tests {
served_names
);
}
#[test]
fn grpc_bind_host_follows_http_tcp_host() {
let mode = HttpListenerMode::BindTcp {
host: "0.0.0.0".to_string(),
port: 8000,
};
assert_eq!(grpc_bind_host(&mode), "0.0.0.0");
}
#[test]
fn grpc_bind_host_defaults_to_loopback_without_tcp_host() {
let unix = HttpListenerMode::BindUnix {
path: "/tmp/vllm.sock".to_string(),
};
let inherited = HttpListenerMode::InheritedFd { fd: 3 };
assert_eq!(grpc_bind_host(&unix), "127.0.0.1");
assert_eq!(grpc_bind_host(&inherited), "127.0.0.1");
}
}