[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Alec
2026-10-02 08:14:37 +00:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent b558f160a2
commit 9d2a6f52b2
11 changed files with 320 additions and 64 deletions
+24 -7
View File
@@ -268,11 +268,6 @@ pub struct SharedRuntimeArgs {
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
#[serde(default)]
pub max_logprobs: Option<i32>,
/// TCP port for the gRPC Inference service. When not set, no gRPC server is
/// started.
#[arg(long)]
#[serde(default)]
pub grpc_port: Option<u16>,
/// Maximum time to wait for active requests to drain during shutdown.
#[arg(long, default_value_t = 0)]
#[serde(default)]
@@ -505,6 +500,7 @@ impl SharedRuntimeArgs {
fn into_bootstrapped_config(
self,
listen_fd: i32,
grpc_listen_fd: Option<i32>,
input_address: String,
output_address: String,
coordinator_address: Option<String>,
@@ -539,6 +535,7 @@ impl SharedRuntimeArgs {
generation_config: self.generation_config,
served_model_name: self.served_model_name,
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
grpc_listener_mode: grpc_listen_fd.map(|fd| HttpListenerMode::InheritedFd { fd }),
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
@@ -555,7 +552,6 @@ impl SharedRuntimeArgs {
tls,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
// The engine is launched and supervised by another process.
manages_engine: false,
@@ -569,6 +565,7 @@ impl SharedRuntimeArgs {
fn into_managed_config(
self,
listener_mode: HttpListenerMode,
grpc_listener_mode: Option<HttpListenerMode>,
handshake_address: String,
advertised_host: String,
engine_count: usize,
@@ -600,6 +597,7 @@ impl SharedRuntimeArgs {
generation_config: self.generation_config,
served_model_name: self.served_model_name,
listener_mode,
grpc_listener_mode,
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
@@ -616,7 +614,6 @@ impl SharedRuntimeArgs {
tls,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
manages_engine,
keep_alive_timeout,
@@ -696,6 +693,10 @@ pub struct FrontendArgs {
/// supervisor.
#[arg(long)]
pub listen_fd: i32,
/// Inherited gRPC listening socket file descriptor passed by the Python
/// supervisor. When not set, no gRPC server is started.
#[arg(long)]
pub grpc_listen_fd: Option<i32>,
/// Frontend input ROUTER socket address that the Python engines will
/// connect to.
#[arg(long)]
@@ -731,6 +732,7 @@ impl FrontendArgs {
let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
self.runtime.into_bootstrapped_config(
self.listen_fd,
self.grpc_listen_fd,
self.input_address,
self.output_address,
self.coordinator_address,
@@ -760,6 +762,10 @@ pub struct ServeArgs {
/// Unix domain socket path. If set, host and port arguments are ignored.
#[arg(long)]
pub uds: Option<String>,
/// TCP port for the gRPC Inference and Control services. When not set, no
/// gRPC server is started.
#[arg(long)]
pub grpc_port: Option<u16>,
/// Flag to print debug information about CLI argument parsing and exit.
#[educe(Debug(ignore))]
@@ -789,9 +795,20 @@ impl ServeArgs {
port: self.port,
},
};
// gRPC follows the HTTP TCP host. With a Unix socket it defaults to IPv4
// loopback rather than all interfaces, so the side-car is never
// accidentally network-exposed.
let grpc_listener_mode = self.grpc_port.map(|port| HttpListenerMode::BindTcp {
host: match self.uds {
Some(_) => "127.0.0.1".to_string(),
None => self.host.clone(),
},
port,
});
self.runtime.clone().into_managed_config(
listener_mode,
grpc_listener_mode,
handshake_address,
self.managed_engine.handshake_host.clone(),
self.managed_engine.data_parallel_size,
+71 -6
View File
@@ -174,6 +174,7 @@ fn serve_args_forward_python_flags_with_separator() {
host: "127.0.0.1",
port: 8000,
uds: None,
grpc_port: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
revision: None,
@@ -188,7 +189,6 @@ fn serve_args_forward_python_flags_with_separator() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1007,6 +1007,7 @@ fn frontend_args_accept_json() {
command: Frontend(
FrontendArgs {
listen_fd: 3,
grpc_listen_fd: None,
input_address: "ipc:///tmp/input.sock",
output_address: "ipc:///tmp/output.sock",
coordinator_address: Some(
@@ -1029,7 +1030,6 @@ fn frontend_args_accept_json() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1687,6 +1687,7 @@ fn serve_args_accept_handshake_aliases() {
host: "127.0.0.1",
port: 8000,
uds: None,
grpc_port: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
revision: None,
@@ -1701,7 +1702,6 @@ fn serve_args_accept_handshake_aliases() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1854,6 +1854,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
host: "127.0.0.1",
port: 8000,
},
grpc_listener_mode: None,
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
@@ -1887,7 +1888,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
tls: None,
api_keys: [],
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: true,
keep_alive_timeout: 5s,
@@ -1949,6 +1949,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
host: "127.0.0.1",
port: 8000,
},
grpc_listener_mode: None,
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
@@ -1982,7 +1983,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
tls: None,
api_keys: [],
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: true,
keep_alive_timeout: 5s,
@@ -2088,6 +2088,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
listener_mode: InheritedFd {
fd: 3,
},
grpc_listener_mode: None,
tool_call_parser: None,
reasoning_parser: None,
tool_strict_level: Auto,
@@ -2121,7 +2122,6 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
tls: None,
api_keys: [],
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: 0ns,
manages_engine: false,
keep_alive_timeout: 5s,
@@ -2139,6 +2139,8 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
"Qwen/Qwen3-0.6B",
"--uds",
"/tmp/vllm.sock",
"--grpc-port",
"50051",
])
.unwrap();
@@ -2153,6 +2155,69 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
path: "/tmp/vllm.sock".to_string(),
}
);
assert_eq!(
config.grpc_listener_mode,
Some(HttpListenerMode::BindTcp {
host: "127.0.0.1".to_string(),
port: 50051,
})
);
}
#[test]
fn serve_frontend_config_binds_grpc_on_http_host() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--host",
"0.0.0.0",
"--grpc-port",
"50051",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
assert_eq!(
config.grpc_listener_mode,
Some(HttpListenerMode::BindTcp {
host: "0.0.0.0".to_string(),
port: 50051,
})
);
}
#[test]
fn frontend_config_adopts_inherited_grpc_listener() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--grpc-listen-fd",
"4",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","grpc_port":50051}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
let config = args.into_config();
assert_eq!(
config.grpc_listener_mode,
Some(HttpListenerMode::InheritedFd { fd: 4 })
);
}
#[test]