diff --git a/rust/README.md b/rust/README.md index 085f70d816..06b50dbbfe 100644 --- a/rust/README.md +++ b/rust/README.md @@ -90,6 +90,27 @@ It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only tokenizer and model configuration files are loaded; model weights, PyTorch, and vLLM kernels are not required. +To serve HTTPS, pass `--ssl-certfile`. If the certificate and private key are +in separate files, pass `--ssl-keyfile` for the key; otherwise the key is read from +the certificate file. For mTLS, pass `--ssl-ca-certs` with a CA +bundle and set `--ssl-cert-reqs` to `1` (optional) or `2` (required) to +verify client certificates. Use `--ssl-ciphers` to override the default +OpenSSL cipher list. + +```bash +# Combined PEM (cert + key in one file): +vllm-rs \ + render Qwen/Qwen3-32B \ + --host 127.0.0.1 --max-model-len 32768 \ + --ssl-certfile /path/to/combined.pem + +# Separate cert and key files: +vllm-rs \ + render Qwen/Qwen3-32B \ + --host 127.0.0.1 --max-model-len 32768 \ + --ssl-certfile /path/to/cert.pem --ssl-keyfile /path/to/key.pem +``` + The render endpoints return the public token-in `GenerateRequest` consumed by the Rust `/inference/v1/generate` endpoint. A chat render response, or one item from a completion render response, can be submitted to that endpoint without diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index a0437679be..8f8208c016 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -7,6 +7,7 @@ //! - Engine args: //! - Environment variables: +mod ssl; mod unsupported; use std::collections::HashMap; @@ -30,9 +31,10 @@ use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_arg use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig, - RendererSelection, TlsConfig, + RendererSelection, }; +use crate::cli::ssl::SslArgs; use crate::cli::unsupported::UnsupportedArgs; /// Top-level parser for the `vllm-rs` binary. @@ -143,10 +145,15 @@ pub struct RenderArgs { /// Maximum accepted logprobs count; -1 disables the cap. #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] max_logprobs: Option, + /// TLS options for HTTPS/mTLS. + #[command(flatten)] + ssl: SslArgs, } impl RenderArgs { pub(super) fn into_config(self) -> RenderConfig { + let tls = self.ssl.tls_config(); + RenderConfig { model: self.model, served_model_name: self.served_model_name, @@ -160,6 +167,7 @@ impl RenderArgs { chat_template_content_format: self.chat_template_content_format, max_model_len: self.max_model_len, max_logprobs: self.max_logprobs, + tls, } } } @@ -360,33 +368,10 @@ pub struct SharedRuntimeArgs { #[serde(default)] pub allow_credentials: bool, - /// The file path to the SSL key file. When omitted, the key is read from - /// `--ssl-certfile` (combined PEM). - #[arg(long)] - #[serde(default)] - pub ssl_keyfile: Option, - - /// The file path to the SSL cert file. Enables TLS when set. - #[arg(long)] - #[serde(default)] - pub ssl_certfile: Option, - - /// The CA certificates file used to verify client certificates (mTLS). - #[arg(long)] - #[serde(default)] - pub ssl_ca_certs: Option, - - /// Whether a client certificate is required: 0 = none, 1 = optional, - /// 2 = required (mirrors Python's `ssl.CERT_*`). - #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))] - #[serde(default)] - pub ssl_cert_reqs: i32, - - /// OpenSSL cipher string for HTTPS (TLS 1.2 and below). - /// When unset, the linked OpenSSL's default suites are used. - #[arg(long)] - #[serde(default)] - pub ssl_ciphers: Option, + /// TLS options for HTTPS/mTLS. + #[command(flatten)] + #[serde(default, flatten)] + pub ssl: SslArgs, /// Profiler configuration forwarded by the Python supervisor. /// @@ -478,7 +463,7 @@ impl SharedRuntimeArgs { let keep_alive_timeout = self.keep_alive_timeout(); let api_server_options = self.api_server_options(); let cors = self.cors_config(); - let tls = self.tls_config(); + let tls = self.ssl.tls_config(); let profiler = self.profiler(); Config { @@ -536,7 +521,7 @@ impl SharedRuntimeArgs { let keep_alive_timeout = self.keep_alive_timeout(); let api_server_options = self.api_server_options(); let cors = self.cors_config(); - let tls = self.tls_config(); + let tls = self.ssl.tls_config(); let profiler = self.profiler(); Config { @@ -591,23 +576,6 @@ impl SharedRuntimeArgs { allow_credentials: self.allow_credentials, } } - - /// Build the TLS config: `Some` when any `ssl_*` argument is set, else - /// `None` (plaintext). The combination is validated in [`Config::validate`]. - fn tls_config(&self) -> Option { - let tls_requested = self.ssl_certfile.is_some() - || self.ssl_keyfile.is_some() - || self.ssl_ca_certs.is_some() - || self.ssl_cert_reqs != 0 - || self.ssl_ciphers.is_some(); - tls_requested.then(|| TlsConfig { - cert_file: self.ssl_certfile.clone(), - key_file: self.ssl_keyfile.clone(), - ca_certs: self.ssl_ca_certs.clone(), - cert_reqs: self.ssl_cert_reqs, - ciphers: self.ssl_ciphers.clone(), - }) - } } fn default_engine_ready_timeout_secs() -> u64 { diff --git a/rust/src/cmd/src/cli/ssl.rs b/rust/src/cmd/src/cli/ssl.rs new file mode 100644 index 0000000000..b167c9ed23 --- /dev/null +++ b/rust/src/cmd/src/cli/ssl.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! TLS CLI options mirroring Python's uvicorn `ssl_*` arguments, shared by the +//! OpenAI frontend and the render server. + +use clap::Args; +use serde::Deserialize; +use vllm_server::TlsConfig; + +/// TLS options mirroring Python's uvicorn `ssl_*` arguments, shared by the +/// OpenAI frontend and the render server. +#[derive(Clone, Debug, Default, Args, PartialEq, Eq, Deserialize)] +pub struct SslArgs { + /// The file path to the SSL key file. When omitted, the key is read from + /// `--ssl-certfile` (combined PEM). + #[arg(long)] + #[serde(default)] + pub ssl_keyfile: Option, + + /// The file path to the SSL cert file. Enables TLS when set. + #[arg(long)] + #[serde(default)] + pub ssl_certfile: Option, + + /// The CA certificates file used to verify client certificates (mTLS). + #[arg(long)] + #[serde(default)] + pub ssl_ca_certs: Option, + + /// Whether a client certificate is required: 0 = none, 1 = optional, + /// 2 = required (mirrors Python's `ssl.CERT_*`). + #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))] + #[serde(default)] + pub ssl_cert_reqs: i32, + + /// OpenSSL cipher string for HTTPS (TLS 1.2 and below). + /// When unset, the linked OpenSSL's default suites are used. + #[arg(long)] + #[serde(default)] + pub ssl_ciphers: Option, +} + +impl SslArgs { + /// Build the TLS config: `Some` when any `ssl_*` argument is set, else + /// `None` (plaintext). The combination is validated in [`TlsConfig::validate`]. + pub fn tls_config(&self) -> Option { + let tls_requested = self.ssl_certfile.is_some() + || self.ssl_keyfile.is_some() + || self.ssl_ca_certs.is_some() + || self.ssl_cert_reqs != 0 + || self.ssl_ciphers.is_some(); + tls_requested.then(|| TlsConfig { + cert_file: self.ssl_certfile.clone(), + key_file: self.ssl_keyfile.clone(), + ca_certs: self.ssl_ca_certs.clone(), + cert_reqs: self.ssl_cert_reqs, + ciphers: self.ssl_ciphers.clone(), + }) + } +} diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index b1a178b670..603c820bfc 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -31,7 +31,7 @@ fn bench_serve_args_parse_without_managed_engine_repartition() { } #[test] -fn render_args_build_supported_config() { +fn render_args_build_config_without_tls() { let cli = Cli::try_parse_from([ "vllm-rs", "render", @@ -63,6 +63,76 @@ fn render_args_build_supported_config() { assert_eq!(config.reasoning_parser, ParserSelection::Auto); assert_eq!(config.renderer, RendererSelection::DeepSeekV32); assert_eq!(config.max_logprobs, Some(-1)); + assert!(config.tls.is_none()); +} + +#[test] +fn render_args_build_config_with_tls() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "render", + "Qwen/Qwen2.5-0.5B-Instruct", + "--max-model-len", + "32768", + "--ssl-certfile", + "/tmp/cert.pem", + "--ssl-keyfile", + "/tmp/key.pem", + ]) + .unwrap(); + + let Command::Render(args) = cli.command else { + panic!("expected render args"); + }; + let config = args.into_config(); + + let tls = config.tls.expect("TLS should be enabled"); + assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem")); + assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem")); +} + +#[test] +fn render_args_reject_tls_without_certificate() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "render", + "Qwen/Qwen2.5-0.5B-Instruct", + "--max-model-len", + "32768", + "--ssl-keyfile", + "/tmp/key.pem", + ]) + .unwrap(); + + let Command::Render(args) = cli.command else { + panic!("expected render args"); + }; + let error = args.into_config().validate().unwrap_err(); + + assert!(error.to_string().contains("--ssl-certfile is required")); +} + +#[test] +fn render_args_reject_client_cert_verification_without_ca() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "render", + "Qwen/Qwen2.5-0.5B-Instruct", + "--max-model-len", + "32768", + "--ssl-certfile", + "/tmp/cert.pem", + "--ssl-cert-reqs", + "2", + ]) + .unwrap(); + + let Command::Render(args) = cli.command else { + panic!("expected render args"); + }; + let error = args.into_config().validate().unwrap_err(); + + assert!(error.to_string().contains("--ssl-ca-certs is required")); } #[test] @@ -127,11 +197,13 @@ fn serve_args_forward_python_flags_with_separator() { ], ), allow_credentials: false, - ssl_keyfile: None, - ssl_certfile: None, - ssl_ca_certs: None, - ssl_cert_reqs: 0, - ssl_ciphers: None, + ssl: SslArgs { + ssl_keyfile: None, + ssl_certfile: None, + ssl_ca_certs: None, + ssl_cert_reqs: 0, + ssl_ciphers: None, + }, profiler_config: None, }, managed_engine: ManagedEngineArgs { @@ -944,11 +1016,13 @@ fn frontend_args_accept_json() { ], ), allow_credentials: false, - ssl_keyfile: None, - ssl_certfile: None, - ssl_ca_certs: None, - ssl_cert_reqs: 0, - ssl_ciphers: None, + ssl: SslArgs { + ssl_keyfile: None, + ssl_certfile: None, + ssl_ca_certs: None, + ssl_cert_reqs: 0, + ssl_ciphers: None, + }, profiler_config: None, }, }, @@ -1539,11 +1613,13 @@ fn serve_args_accept_handshake_aliases() { ], ), allow_credentials: false, - ssl_keyfile: None, - ssl_certfile: None, - ssl_ca_certs: None, - ssl_cert_reqs: 0, - ssl_ciphers: None, + ssl: SslArgs { + ssl_keyfile: None, + ssl_certfile: None, + ssl_ca_certs: None, + ssl_cert_reqs: 0, + ssl_ciphers: None, + }, profiler_config: None, }, managed_engine: ManagedEngineArgs { diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 35c821bc6e..5f5e8cdc99 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -14,7 +14,11 @@ use vllm_chat::{ }; use vllm_text::TextRequestProcessor; -use crate::{HttpListenerMode, listener::Listener}; +use crate::{ + HttpListenerMode, TlsConfig, + listener::{Listener, MaybeTlsListener}, + tls, +}; /// Configuration for the engine-free text rendering server. #[derive(Debug)] @@ -31,6 +35,7 @@ pub struct RenderConfig { pub chat_template_content_format: ChatTemplateContentFormatOption, pub max_model_len: u32, pub max_logprobs: Option, + pub tls: Option, } impl RenderConfig { @@ -41,6 +46,9 @@ impl RenderConfig { if self.max_logprobs.is_some_and(|value| value < -1) { bail!("max_logprobs must be non-negative or -1"); } + if let Some(tls) = &self.tls { + tls.validate()?; + } Ok(()) } } @@ -87,6 +95,12 @@ async fn build_state(config: &RenderConfig) -> Result> { /// to an inference engine. pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> { config.validate().context("invalid render server configuration")?; + let tls_config = config + .tls + .as_ref() + .map(tls::build_server_config) + .transpose() + .context("invalid TLS configuration")?; let state = tokio::select! { result = build_state(&config) => result?, _ = shutdown.cancelled() => return Ok(()), @@ -99,8 +113,18 @@ pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> .await .with_context(|| format!("failed to bind {}:{}", config.host, config.port))?; let address = listener.local_addr_display()?; + let scheme = if tls_config.is_some() { + "https" + } else { + "http" + }; + let listener = match tls_config { + Some(context) => MaybeTlsListener::tls(listener, context), + None => MaybeTlsListener::plain(listener), + }; info!( %address, + scheme, model = %config.model, "starting engine-free Rust render server" ); @@ -132,6 +156,7 @@ mod tests { chat_template_content_format: ChatTemplateContentFormatOption::Auto, max_model_len: 128, max_logprobs: None, + tls: None, }, shutdown, )