iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Add support for TLS in render server (#54999)

Browse Source 
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Richard <noreply@moonshot.cn>
Signed-off-by: Wen Zhou <wenzhou@redhat.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Wen Zhou
2026-09-03 21:56:44 +00:00
committed by GitHub
co-authored by Bugen Zhao Richard
parent 6fdee17f84
commit c7e6e36fa9
5 changed files with 215 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/README.md
+21
View File
@@ -90,6 +90,27 @@ It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only
tokenizer and model configuration files are loaded; model weights, PyTorch,
and vLLM kernels are not required.
To serve HTTPS, pass `--ssl-certfile`. If the certificate and private key are
in separate files, pass `--ssl-keyfile` for the key; otherwise the key is read from
the certificate file. For mTLS, pass `--ssl-ca-certs` with a CA
bundle and set `--ssl-cert-reqs` to `1` (optional) or `2` (required) to
verify client certificates. Use `--ssl-ciphers` to override the default
OpenSSL cipher list.
```bash
# Combined PEM (cert + key in one file):
vllm-rs \
render Qwen/Qwen3-32B \
--host 127.0.0.1 --max-model-len 32768 \
--ssl-certfile /path/to/combined.pem
# Separate cert and key files:
vllm-rs \
render Qwen/Qwen3-32B \
--host 127.0.0.1 --max-model-len 32768 \
--ssl-certfile /path/to/cert.pem --ssl-keyfile /path/to/key.pem
```
The render endpoints return the public token-in `GenerateRequest` consumed by
the Rust `/inference/v1/generate` endpoint. A chat render response, or one item
from a completion render response, can be submitted to that endpoint without
rust/src/cmd/src/cli.rs
+15 -47
View File
@@ -7,6 +7,7 @@
//! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
//! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
mod ssl;
mod unsupported;
use std::collections::HashMap;
@@ -30,9 +31,10 @@ use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_arg
use vllm_server::{
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
RendererSelection, TlsConfig,
RendererSelection,
};
use crate::cli::ssl::SslArgs;
use crate::cli::unsupported::UnsupportedArgs;
/// Top-level parser for the `vllm-rs` binary.
@@ -143,10 +145,15 @@ pub struct RenderArgs {
/// Maximum accepted logprobs count; -1 disables the cap.
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
max_logprobs: Option<i32>,
/// TLS options for HTTPS/mTLS.
#[command(flatten)]
ssl: SslArgs,
}
impl RenderArgs {
pub(super) fn into_config(self) -> RenderConfig {
let tls = self.ssl.tls_config();
RenderConfig {
model: self.model,
served_model_name: self.served_model_name,
@@ -160,6 +167,7 @@ impl RenderArgs {
chat_template_content_format: self.chat_template_content_format,
max_model_len: self.max_model_len,
max_logprobs: self.max_logprobs,
tls,
}
}
}
@@ -360,33 +368,10 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub allow_credentials: bool,
/// The file path to the SSL key file. When omitted, the key is read from
/// `--ssl-certfile` (combined PEM).
#[arg(long)]
#[serde(default)]
pub ssl_keyfile: Option<String>,
/// The file path to the SSL cert file. Enables TLS when set.
#[arg(long)]
#[serde(default)]
pub ssl_certfile: Option<String>,
/// The CA certificates file used to verify client certificates (mTLS).
#[arg(long)]
#[serde(default)]
pub ssl_ca_certs: Option<String>,
/// Whether a client certificate is required: 0 = none, 1 = optional,
/// 2 = required (mirrors Python's `ssl.CERT_*`).
#[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
#[serde(default)]
pub ssl_cert_reqs: i32,
/// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
/// When unset, the linked OpenSSL's default suites are used.
#[arg(long)]
#[serde(default)]
pub ssl_ciphers: Option<String>,
/// TLS options for HTTPS/mTLS.
#[command(flatten)]
#[serde(default, flatten)]
pub ssl: SslArgs,
/// Profiler configuration forwarded by the Python supervisor.
///
@@ -478,7 +463,7 @@ impl SharedRuntimeArgs {
let keep_alive_timeout = self.keep_alive_timeout();
let api_server_options = self.api_server_options();
let cors = self.cors_config();
let tls = self.tls_config();
let tls = self.ssl.tls_config();
let profiler = self.profiler();
Config {
@@ -536,7 +521,7 @@ impl SharedRuntimeArgs {
let keep_alive_timeout = self.keep_alive_timeout();
let api_server_options = self.api_server_options();
let cors = self.cors_config();
let tls = self.tls_config();
let tls = self.ssl.tls_config();
let profiler = self.profiler();
Config {
@@ -591,23 +576,6 @@ impl SharedRuntimeArgs {
allow_credentials: self.allow_credentials,
}
}
/// Build the TLS config: `Some` when any `ssl_*` argument is set, else
/// `None` (plaintext). The combination is validated in [`Config::validate`].
fn tls_config(&self) -> Option<TlsConfig> {
let tls_requested = self.ssl_certfile.is_some()
|| self.ssl_keyfile.is_some()
|| self.ssl_ca_certs.is_some()
|| self.ssl_cert_reqs != 0
|| self.ssl_ciphers.is_some();
tls_requested.then(|| TlsConfig {
cert_file: self.ssl_certfile.clone(),
key_file: self.ssl_keyfile.clone(),
ca_certs: self.ssl_ca_certs.clone(),
cert_reqs: self.ssl_cert_reqs,
ciphers: self.ssl_ciphers.clone(),
})
}
}
fn default_engine_ready_timeout_secs() -> u64 {
rust/src/cmd/src/cli/ssl.rs
+61
View File
@@ -0,0 +1,61 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! TLS CLI options mirroring Python's uvicorn `ssl_*` arguments, shared by the
//! OpenAI frontend and the render server.
use clap::Args;
use serde::Deserialize;
use vllm_server::TlsConfig;
/// TLS options mirroring Python's uvicorn `ssl_*` arguments, shared by the
/// OpenAI frontend and the render server.
#[derive(Clone, Debug, Default, Args, PartialEq, Eq, Deserialize)]
pub struct SslArgs {
/// The file path to the SSL key file. When omitted, the key is read from
/// `--ssl-certfile` (combined PEM).
#[arg(long)]
#[serde(default)]
pub ssl_keyfile: Option<String>,
/// The file path to the SSL cert file. Enables TLS when set.
#[arg(long)]
#[serde(default)]
pub ssl_certfile: Option<String>,
/// The CA certificates file used to verify client certificates (mTLS).
#[arg(long)]
#[serde(default)]
pub ssl_ca_certs: Option<String>,
/// Whether a client certificate is required: 0 = none, 1 = optional,
/// 2 = required (mirrors Python's `ssl.CERT_*`).
#[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
#[serde(default)]
pub ssl_cert_reqs: i32,
/// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
/// When unset, the linked OpenSSL's default suites are used.
#[arg(long)]
#[serde(default)]
pub ssl_ciphers: Option<String>,
}
impl SslArgs {
/// Build the TLS config: `Some` when any `ssl_*` argument is set, else
/// `None` (plaintext). The combination is validated in [`TlsConfig::validate`].
pub fn tls_config(&self) -> Option<TlsConfig> {
let tls_requested = self.ssl_certfile.is_some()
|| self.ssl_keyfile.is_some()
|| self.ssl_ca_certs.is_some()
|| self.ssl_cert_reqs != 0
|| self.ssl_ciphers.is_some();
tls_requested.then(|| TlsConfig {
cert_file: self.ssl_certfile.clone(),
key_file: self.ssl_keyfile.clone(),
ca_certs: self.ssl_ca_certs.clone(),
cert_reqs: self.ssl_cert_reqs,
ciphers: self.ssl_ciphers.clone(),
})
}
}
rust/src/cmd/src/cli/tests.rs
+92 -16
View File
@@ -31,7 +31,7 @@ fn bench_serve_args_parse_without_managed_engine_repartition() {
}
#[test]
fn render_args_build_supported_config() {
fn render_args_build_config_without_tls() {
let cli = Cli::try_parse_from([
"vllm-rs",
"render",
@@ -63,6 +63,76 @@ fn render_args_build_supported_config() {
assert_eq!(config.reasoning_parser, ParserSelection::Auto);
assert_eq!(config.renderer, RendererSelection::DeepSeekV32);
assert_eq!(config.max_logprobs, Some(-1));
assert!(config.tls.is_none());
}
#[test]
fn render_args_build_config_with_tls() {
let cli = Cli::try_parse_from([
"vllm-rs",
"render",
"Qwen/Qwen2.5-0.5B-Instruct",
"--max-model-len",
"32768",
"--ssl-certfile",
"/tmp/cert.pem",
"--ssl-keyfile",
"/tmp/key.pem",
])
.unwrap();
let Command::Render(args) = cli.command else {
panic!("expected render args");
};
let config = args.into_config();
let tls = config.tls.expect("TLS should be enabled");
assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
}
#[test]
fn render_args_reject_tls_without_certificate() {
let cli = Cli::try_parse_from([
"vllm-rs",
"render",
"Qwen/Qwen2.5-0.5B-Instruct",
"--max-model-len",
"32768",
"--ssl-keyfile",
"/tmp/key.pem",
])
.unwrap();
let Command::Render(args) = cli.command else {
panic!("expected render args");
};
let error = args.into_config().validate().unwrap_err();
assert!(error.to_string().contains("--ssl-certfile is required"));
}
#[test]
fn render_args_reject_client_cert_verification_without_ca() {
let cli = Cli::try_parse_from([
"vllm-rs",
"render",
"Qwen/Qwen2.5-0.5B-Instruct",
"--max-model-len",
"32768",
"--ssl-certfile",
"/tmp/cert.pem",
"--ssl-cert-reqs",
"2",
])
.unwrap();
let Command::Render(args) = cli.command else {
panic!("expected render args");
};
let error = args.into_config().validate().unwrap_err();
assert!(error.to_string().contains("--ssl-ca-certs is required"));
}
#[test]
@@ -127,11 +197,13 @@ fn serve_args_forward_python_flags_with_separator() {
],
),
allow_credentials: false,
ssl_keyfile: None,
ssl_certfile: None,
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
ssl: SslArgs {
ssl_keyfile: None,
ssl_certfile: None,
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
},
profiler_config: None,
},
managed_engine: ManagedEngineArgs {
@@ -944,11 +1016,13 @@ fn frontend_args_accept_json() {
],
),
allow_credentials: false,
ssl_keyfile: None,
ssl_certfile: None,
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
ssl: SslArgs {
ssl_keyfile: None,
ssl_certfile: None,
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
},
profiler_config: None,
},
},
@@ -1539,11 +1613,13 @@ fn serve_args_accept_handshake_aliases() {
],
),
allow_credentials: false,
ssl_keyfile: None,
ssl_certfile: None,
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
ssl: SslArgs {
ssl_keyfile: None,
ssl_certfile: None,
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
},
profiler_config: None,
},
managed_engine: ManagedEngineArgs {
rust/src/server/src/render.rs
+26 -1
View File
@@ -14,7 +14,11 @@ use vllm_chat::{
};
use vllm_text::TextRequestProcessor;
use crate::{HttpListenerMode, listener::Listener};
use crate::{
HttpListenerMode, TlsConfig,
listener::{Listener, MaybeTlsListener},
tls,
};
/// Configuration for the engine-free text rendering server.
#[derive(Debug)]
@@ -31,6 +35,7 @@ pub struct RenderConfig {
pub chat_template_content_format: ChatTemplateContentFormatOption,
pub max_model_len: u32,
pub max_logprobs: Option<i32>,
pub tls: Option<TlsConfig>,
}
impl RenderConfig {
@@ -41,6 +46,9 @@ impl RenderConfig {
if self.max_logprobs.is_some_and(|value| value < -1) {
bail!("max_logprobs must be non-negative or -1");
}
if let Some(tls) = &self.tls {
tls.validate()?;
}
Ok(())
}
}
@@ -87,6 +95,12 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
/// to an inference engine.
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
config.validate().context("invalid render server configuration")?;
let tls_config = config
.tls
.as_ref()
.map(tls::build_server_config)
.transpose()
.context("invalid TLS configuration")?;
let state = tokio::select! {
result = build_state(&config) => result?,
_ = shutdown.cancelled() => return Ok(()),
@@ -99,8 +113,18 @@ pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) ->
.await
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
let address = listener.local_addr_display()?;
let scheme = if tls_config.is_some() {
"https"
} else {
"http"
};
let listener = match tls_config {
Some(context) => MaybeTlsListener::tls(listener, context),
None => MaybeTlsListener::plain(listener),
};
info!(
%address,
scheme,
model = %config.model,
"starting engine-free Rust render server"
);
@@ -132,6 +156,7 @@ mod tests {
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_model_len: 128,
max_logprobs: None,
tls: None,
},
shutdown,
)