[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Alec Flowers <aflowers@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
b558f160a2
commit
9d2a6f52b2
+24
-7
@@ -268,11 +268,6 @@ pub struct SharedRuntimeArgs {
|
||||
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
||||
#[serde(default)]
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// TCP port for the gRPC Inference service. When not set, no gRPC server is
|
||||
/// started.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active requests to drain during shutdown.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
@@ -505,6 +500,7 @@ impl SharedRuntimeArgs {
|
||||
fn into_bootstrapped_config(
|
||||
self,
|
||||
listen_fd: i32,
|
||||
grpc_listen_fd: Option<i32>,
|
||||
input_address: String,
|
||||
output_address: String,
|
||||
coordinator_address: Option<String>,
|
||||
@@ -539,6 +535,7 @@ impl SharedRuntimeArgs {
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
grpc_listener_mode: grpc_listen_fd.map(|fd| HttpListenerMode::InheritedFd { fd }),
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
@@ -555,7 +552,6 @@ impl SharedRuntimeArgs {
|
||||
tls,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
// The engine is launched and supervised by another process.
|
||||
manages_engine: false,
|
||||
@@ -569,6 +565,7 @@ impl SharedRuntimeArgs {
|
||||
fn into_managed_config(
|
||||
self,
|
||||
listener_mode: HttpListenerMode,
|
||||
grpc_listener_mode: Option<HttpListenerMode>,
|
||||
handshake_address: String,
|
||||
advertised_host: String,
|
||||
engine_count: usize,
|
||||
@@ -600,6 +597,7 @@ impl SharedRuntimeArgs {
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
grpc_listener_mode,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
@@ -616,7 +614,6 @@ impl SharedRuntimeArgs {
|
||||
tls,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
manages_engine,
|
||||
keep_alive_timeout,
|
||||
@@ -696,6 +693,10 @@ pub struct FrontendArgs {
|
||||
/// supervisor.
|
||||
#[arg(long)]
|
||||
pub listen_fd: i32,
|
||||
/// Inherited gRPC listening socket file descriptor passed by the Python
|
||||
/// supervisor. When not set, no gRPC server is started.
|
||||
#[arg(long)]
|
||||
pub grpc_listen_fd: Option<i32>,
|
||||
/// Frontend input ROUTER socket address that the Python engines will
|
||||
/// connect to.
|
||||
#[arg(long)]
|
||||
@@ -731,6 +732,7 @@ impl FrontendArgs {
|
||||
let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
|
||||
self.runtime.into_bootstrapped_config(
|
||||
self.listen_fd,
|
||||
self.grpc_listen_fd,
|
||||
self.input_address,
|
||||
self.output_address,
|
||||
self.coordinator_address,
|
||||
@@ -760,6 +762,10 @@ pub struct ServeArgs {
|
||||
/// Unix domain socket path. If set, host and port arguments are ignored.
|
||||
#[arg(long)]
|
||||
pub uds: Option<String>,
|
||||
/// TCP port for the gRPC Inference and Control services. When not set, no
|
||||
/// gRPC server is started.
|
||||
#[arg(long)]
|
||||
pub grpc_port: Option<u16>,
|
||||
|
||||
/// Flag to print debug information about CLI argument parsing and exit.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -789,9 +795,20 @@ impl ServeArgs {
|
||||
port: self.port,
|
||||
},
|
||||
};
|
||||
// gRPC follows the HTTP TCP host. With a Unix socket it defaults to IPv4
|
||||
// loopback rather than all interfaces, so the side-car is never
|
||||
// accidentally network-exposed.
|
||||
let grpc_listener_mode = self.grpc_port.map(|port| HttpListenerMode::BindTcp {
|
||||
host: match self.uds {
|
||||
Some(_) => "127.0.0.1".to_string(),
|
||||
None => self.host.clone(),
|
||||
},
|
||||
port,
|
||||
});
|
||||
|
||||
self.runtime.clone().into_managed_config(
|
||||
listener_mode,
|
||||
grpc_listener_mode,
|
||||
handshake_address,
|
||||
self.managed_engine.handshake_host.clone(),
|
||||
self.managed_engine.data_parallel_size,
|
||||
|
||||
@@ -174,6 +174,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
uds: None,
|
||||
grpc_port: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
@@ -188,7 +189,6 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
@@ -1007,6 +1007,7 @@ fn frontend_args_accept_json() {
|
||||
command: Frontend(
|
||||
FrontendArgs {
|
||||
listen_fd: 3,
|
||||
grpc_listen_fd: None,
|
||||
input_address: "ipc:///tmp/input.sock",
|
||||
output_address: "ipc:///tmp/output.sock",
|
||||
coordinator_address: Some(
|
||||
@@ -1029,7 +1030,6 @@ fn frontend_args_accept_json() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
@@ -1687,6 +1687,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
uds: None,
|
||||
grpc_port: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
@@ -1701,7 +1702,6 @@ fn serve_args_accept_handshake_aliases() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
@@ -1854,6 +1854,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
},
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
@@ -1887,7 +1888,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
tls: None,
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
manages_engine: true,
|
||||
keep_alive_timeout: 5s,
|
||||
@@ -1949,6 +1949,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
host: "127.0.0.1",
|
||||
port: 8000,
|
||||
},
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
@@ -1982,7 +1983,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
tls: None,
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
manages_engine: true,
|
||||
keep_alive_timeout: 5s,
|
||||
@@ -2088,6 +2088,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
listener_mode: InheritedFd {
|
||||
fd: 3,
|
||||
},
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
@@ -2121,7 +2122,6 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
tls: None,
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
manages_engine: false,
|
||||
keep_alive_timeout: 5s,
|
||||
@@ -2139,6 +2139,8 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--uds",
|
||||
"/tmp/vllm.sock",
|
||||
"--grpc-port",
|
||||
"50051",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -2153,6 +2155,69 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
|
||||
path: "/tmp/vllm.sock".to_string(),
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
config.grpc_listener_mode,
|
||||
Some(HttpListenerMode::BindTcp {
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 50051,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_frontend_config_binds_grpc_on_http_host() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--grpc-port",
|
||||
"50051",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
|
||||
|
||||
assert_eq!(
|
||||
config.grpc_listener_mode,
|
||||
Some(HttpListenerMode::BindTcp {
|
||||
host: "0.0.0.0".to_string(),
|
||||
port: 50051,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_config_adopts_inherited_grpc_listener() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--grpc-listen-fd",
|
||||
"4",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","grpc_port":50051}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
let config = args.into_config();
|
||||
|
||||
assert_eq!(
|
||||
config.grpc_listener_mode,
|
||||
Some(HttpListenerMode::InheritedFd { fd: 4 })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user