[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Alec Flowers <aflowers@nvidia.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Alec
2026-10-02 08:14:37 +00:00
committed by GitHub
co-authored by Bugen Zhao Claude Opus 5.5
parent b558f160a2
commit 9d2a6f52b2
11 changed files with 320 additions and 64 deletions
@@ -69,6 +69,7 @@ async fn main() -> Result<()> {
host: "127.0.0.1".to_string(),
port,
},
grpc_listener_mode: None,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
@@ -85,7 +86,6 @@ async fn main() -> Result<()> {
tls: None,
api_keys: Vec::new(),
disable_log_stats: false,
grpc_port: None,
shutdown_timeout: Duration::ZERO,
manages_engine: false,
keep_alive_timeout: Duration::from_secs(5),
+3 -4
View File
@@ -23,7 +23,7 @@ use vllm_text::backend::hf::HfOverrides;
/// when keep-alive is disabled (`0`).
pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
/// How the HTTP server obtains its listening socket.
/// How the HTTP or gRPC server obtains its listening socket.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub enum HttpListenerMode {
/// Bind a fresh TCP listener on the given host/port.
@@ -229,6 +229,8 @@ pub struct Config {
pub served_model_name: Vec<String>,
/// HTTP listener setup.
pub listener_mode: HttpListenerMode,
/// gRPC listener setup. When `None`, no gRPC server is started.
pub grpc_listener_mode: Option<HttpListenerMode>,
/// Tool-call parser selection.
pub tool_call_parser: ParserSelection,
/// Reasoning parser selection.
@@ -272,9 +274,6 @@ pub struct Config {
/// engine-reported scheduler stats and request lifecycle events are not
/// exported.
pub disable_log_stats: bool,
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
/// started.
pub grpc_port: Option<u16>,
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
pub shutdown_timeout: Duration,
/// Whether the caller manages the engine process and shuts it down when
+4 -37
View File
@@ -37,7 +37,6 @@ use hyper_util::rt::{TokioIo, TokioTimer};
use hyper_util::server::graceful::GracefulShutdown;
use hyper_util::service::TowerToHyperService;
pub use render::{RenderConfig, serve_render};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
use tokio_util::sync::CancellationToken;
use tonic::transport::Server as TonicServer;
@@ -74,17 +73,6 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve
}
}
/// Choose the gRPC listener host. It follows the HTTP TCP host when there is
/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback
/// rather than all interfaces, so the side-car is never accidentally
/// network-exposed.
fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
match listener_mode {
HttpListenerMode::BindTcp { host, .. } => host.as_str(),
HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1",
}
}
/// Build the shared application state for one configured model and one engine
/// client, including any static LoRA adapters from `--lora-modules`.
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
@@ -229,13 +217,11 @@ where
// Optionally bind the gRPC Inference server on a separate port. Bind
// synchronously here so bind errors (port in use, permission denied, ...)
// surface before serving rather than being deferred until shutdown.
let grpc_setup = if let Some(grpc_port) = config.grpc_port {
let grpc_host = grpc_bind_host(&config.listener_mode);
let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
let grpc_setup = if let Some(grpc_listener_mode) = &config.grpc_listener_mode {
let grpc_listener = Listener::bind(grpc_listener_mode)
.await
.with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
let addr = grpc_listener.local_addr()?;
let grpc_listener = Listener::Tcp(grpc_listener);
.with_context(|| format!("failed to bind gRPC listener for {grpc_listener_mode:?}"))?;
let addr = grpc_listener.local_addr_display()?;
// gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2.
let grpc_tls = config
.tls
@@ -479,23 +465,4 @@ mod tests {
served_names
);
}
#[test]
fn grpc_bind_host_follows_http_tcp_host() {
let mode = HttpListenerMode::BindTcp {
host: "0.0.0.0".to_string(),
port: 8000,
};
assert_eq!(grpc_bind_host(&mode), "0.0.0.0");
}
#[test]
fn grpc_bind_host_defaults_to_loopback_without_tcp_host() {
let unix = HttpListenerMode::BindUnix {
path: "/tmp/vllm.sock".to_string(),
};
let inherited = HttpListenerMode::InheritedFd { fd: 3 };
assert_eq!(grpc_bind_host(&unix), "127.0.0.1");
assert_eq!(grpc_bind_host(&inherited), "127.0.0.1");
}
}