[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Alec Flowers <aflowers@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Claude Opus 5.5
parent
b558f160a2
commit
9d2a6f52b2
@@ -69,6 +69,7 @@ async fn main() -> Result<()> {
|
||||
host: "127.0.0.1".to_string(),
|
||||
port,
|
||||
},
|
||||
grpc_listener_mode: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
@@ -85,7 +86,6 @@ async fn main() -> Result<()> {
|
||||
tls: None,
|
||||
api_keys: Vec::new(),
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: Duration::ZERO,
|
||||
manages_engine: false,
|
||||
keep_alive_timeout: Duration::from_secs(5),
|
||||
|
||||
@@ -23,7 +23,7 @@ use vllm_text::backend::hf::HfOverrides;
|
||||
/// when keep-alive is disabled (`0`).
|
||||
pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// How the HTTP server obtains its listening socket.
|
||||
/// How the HTTP or gRPC server obtains its listening socket.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub enum HttpListenerMode {
|
||||
/// Bind a fresh TCP listener on the given host/port.
|
||||
@@ -229,6 +229,8 @@ pub struct Config {
|
||||
pub served_model_name: Vec<String>,
|
||||
/// HTTP listener setup.
|
||||
pub listener_mode: HttpListenerMode,
|
||||
/// gRPC listener setup. When `None`, no gRPC server is started.
|
||||
pub grpc_listener_mode: Option<HttpListenerMode>,
|
||||
/// Tool-call parser selection.
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
@@ -272,9 +274,6 @@ pub struct Config {
|
||||
/// engine-reported scheduler stats and request lifecycle events are not
|
||||
/// exported.
|
||||
pub disable_log_stats: bool,
|
||||
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
|
||||
/// started.
|
||||
pub grpc_port: Option<u16>,
|
||||
/// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
|
||||
pub shutdown_timeout: Duration,
|
||||
/// Whether the caller manages the engine process and shuts it down when
|
||||
|
||||
@@ -37,7 +37,6 @@ use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use hyper_util::server::graceful::GracefulShutdown;
|
||||
use hyper_util::service::TowerToHyperService;
|
||||
pub use render::{RenderConfig, serve_render};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::transport::Server as TonicServer;
|
||||
@@ -74,17 +73,6 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve
|
||||
}
|
||||
}
|
||||
|
||||
/// Choose the gRPC listener host. It follows the HTTP TCP host when there is
|
||||
/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback
|
||||
/// rather than all interfaces, so the side-car is never accidentally
|
||||
/// network-exposed.
|
||||
fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
|
||||
match listener_mode {
|
||||
HttpListenerMode::BindTcp { host, .. } => host.as_str(),
|
||||
HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the shared application state for one configured model and one engine
|
||||
/// client, including any static LoRA adapters from `--lora-modules`.
|
||||
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
@@ -229,13 +217,11 @@ where
|
||||
// Optionally bind the gRPC Inference server on a separate port. Bind
|
||||
// synchronously here so bind errors (port in use, permission denied, ...)
|
||||
// surface before serving rather than being deferred until shutdown.
|
||||
let grpc_setup = if let Some(grpc_port) = config.grpc_port {
|
||||
let grpc_host = grpc_bind_host(&config.listener_mode);
|
||||
let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
|
||||
let grpc_setup = if let Some(grpc_listener_mode) = &config.grpc_listener_mode {
|
||||
let grpc_listener = Listener::bind(grpc_listener_mode)
|
||||
.await
|
||||
.with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
|
||||
let addr = grpc_listener.local_addr()?;
|
||||
let grpc_listener = Listener::Tcp(grpc_listener);
|
||||
.with_context(|| format!("failed to bind gRPC listener for {grpc_listener_mode:?}"))?;
|
||||
let addr = grpc_listener.local_addr_display()?;
|
||||
// gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2.
|
||||
let grpc_tls = config
|
||||
.tls
|
||||
@@ -479,23 +465,4 @@ mod tests {
|
||||
served_names
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_bind_host_follows_http_tcp_host() {
|
||||
let mode = HttpListenerMode::BindTcp {
|
||||
host: "0.0.0.0".to_string(),
|
||||
port: 8000,
|
||||
};
|
||||
assert_eq!(grpc_bind_host(&mode), "0.0.0.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_bind_host_defaults_to_loopback_without_tcp_host() {
|
||||
let unix = HttpListenerMode::BindUnix {
|
||||
path: "/tmp/vllm.sock".to_string(),
|
||||
};
|
||||
let inherited = HttpListenerMode::InheritedFd { fd: 3 };
|
||||
assert_eq!(grpc_bind_host(&unix), "127.0.0.1");
|
||||
assert_eq!(grpc_bind_host(&inherited), "127.0.0.1");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user