[Rust Frontend] Add static HTTPS and mTLS support for HTTP and gRPC (#45890)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
43916891b2
commit
b8cb75b149
+66
-1
@@ -25,7 +25,7 @@ use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
use vllm_server::{
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||
HttpListenerMode, ParserSelection, RendererSelection,
|
||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
|
||||
};
|
||||
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
@@ -154,6 +154,11 @@ pub struct SharedRuntimeArgs {
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
pub shutdown_timeout: u64,
|
||||
/// Maximum idle time (seconds) on a keep-alive HTTP connection before the
|
||||
/// server closes it (default 5).
|
||||
#[arg(long = "http-timeout-keep-alive", env = "VLLM_HTTP_TIMEOUT_KEEP_ALIVE")]
|
||||
#[serde(default)]
|
||||
pub http_timeout_keep_alive: Option<u64>,
|
||||
|
||||
/// The file path to the chat template, or the template in single-line form
|
||||
/// for the specified model.
|
||||
@@ -257,6 +262,34 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub allow_credentials: bool,
|
||||
|
||||
/// The file path to the SSL key file. When omitted, the key is read from
|
||||
/// `--ssl-certfile` (combined PEM).
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub ssl_keyfile: Option<String>,
|
||||
|
||||
/// The file path to the SSL cert file. Enables TLS when set.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub ssl_certfile: Option<String>,
|
||||
|
||||
/// The CA certificates file used to verify client certificates (mTLS).
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub ssl_ca_certs: Option<String>,
|
||||
|
||||
/// Whether a client certificate is required: 0 = none, 1 = optional,
|
||||
/// 2 = required (mirrors Python's `ssl.CERT_*`).
|
||||
#[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
|
||||
#[serde(default)]
|
||||
pub ssl_cert_reqs: i32,
|
||||
|
||||
/// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
|
||||
/// When unset, the linked OpenSSL's default suites are used.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub ssl_ciphers: Option<String>,
|
||||
|
||||
/// Unsupported Python vLLM frontend arguments recognized but not yet
|
||||
/// implemented in Rust.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -277,6 +310,13 @@ impl SharedRuntimeArgs {
|
||||
Duration::from_secs(self.shutdown_timeout)
|
||||
}
|
||||
|
||||
/// Maximum idle time on a keep-alive HTTP connection before the server
|
||||
/// closes it.
|
||||
pub fn keep_alive_timeout(&self) -> Duration {
|
||||
self.http_timeout_keep_alive
|
||||
.map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
|
||||
}
|
||||
|
||||
/// Apply fallback logic for API key configuration from env variables.
|
||||
fn apply_env_api_key_fallback(&mut self) {
|
||||
if self.api_key.is_empty()
|
||||
@@ -301,8 +341,10 @@ impl SharedRuntimeArgs {
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
let keep_alive_timeout = self.keep_alive_timeout();
|
||||
let api_server_options = self.api_server_options();
|
||||
let cors = self.cors_config();
|
||||
let tls = self.tls_config();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::Bootstrapped {
|
||||
@@ -329,10 +371,12 @@ impl SharedRuntimeArgs {
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
cors,
|
||||
tls,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
keep_alive_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -349,8 +393,10 @@ impl SharedRuntimeArgs {
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
let keep_alive_timeout = self.keep_alive_timeout();
|
||||
let api_server_options = self.api_server_options();
|
||||
let cors = self.cors_config();
|
||||
let tls = self.tls_config();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
@@ -375,10 +421,12 @@ impl SharedRuntimeArgs {
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
cors,
|
||||
tls,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
keep_alive_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -398,6 +446,23 @@ impl SharedRuntimeArgs {
|
||||
allow_credentials: self.allow_credentials,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the TLS config: `Some` when any `ssl_*` argument is set, else
|
||||
/// `None` (plaintext). The combination is validated in [`Config::validate`].
|
||||
fn tls_config(&self) -> Option<TlsConfig> {
|
||||
let tls_requested = self.ssl_certfile.is_some()
|
||||
|| self.ssl_keyfile.is_some()
|
||||
|| self.ssl_ca_certs.is_some()
|
||||
|| self.ssl_cert_reqs != 0
|
||||
|| self.ssl_ciphers.is_some();
|
||||
tls_requested.then(|| TlsConfig {
|
||||
cert_file: self.ssl_certfile.clone(),
|
||||
key_file: self.ssl_keyfile.clone(),
|
||||
ca_certs: self.ssl_ca_certs.clone(),
|
||||
cert_reqs: self.ssl_cert_reqs,
|
||||
ciphers: self.ssl_ciphers.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn default_engine_ready_timeout_secs() -> u64 {
|
||||
|
||||
@@ -41,6 +41,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
@@ -65,6 +66,11 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
],
|
||||
),
|
||||
allow_credentials: false,
|
||||
ssl_keyfile: None,
|
||||
ssl_certfile: None,
|
||||
ssl_ca_certs: None,
|
||||
ssl_cert_reqs: 0,
|
||||
ssl_ciphers: None,
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "../vllm/.venv/bin/python",
|
||||
@@ -363,6 +369,140 @@ fn serve_passes_enable_prompt_tokens_details_into_config() {
|
||||
assert!(config.api_server_options.enable_prompt_tokens_details);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_passes_tls_into_config() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--ssl-certfile",
|
||||
"/tmp/cert.pem",
|
||||
"--ssl-keyfile",
|
||||
"/tmp/key.pem",
|
||||
"--ssl-ca-certs",
|
||||
"/tmp/ca.pem",
|
||||
"--ssl-cert-reqs",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
let tls = config.tls.expect("tls configured");
|
||||
assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
|
||||
assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
|
||||
assert_eq!(tls.ca_certs.as_deref(), Some("/tmp/ca.pem"));
|
||||
assert_eq!(tls.cert_reqs, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_without_ssl_flags_has_no_tls() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
assert!(config.tls.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_ssl_keyfile_without_certfile_fails_validation() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--ssl-keyfile",
|
||||
"/tmp/key.pem",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
// TLS is requested (a key was given) but there is no certificate, so
|
||||
// validation fails loud rather than silently serving plaintext.
|
||||
assert_eq!(config.tls.as_ref().expect("tls requested").cert_file, None);
|
||||
let err = config.validate().unwrap_err().to_string();
|
||||
assert!(err.contains("--ssl-certfile is required"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_mtls_without_ca_certs_fails_validation() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--ssl-certfile",
|
||||
"/tmp/cert.pem",
|
||||
"--ssl-cert-reqs",
|
||||
"2",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
// Client-cert verification without a CA bundle has nothing to verify
|
||||
// against, so it fails loud at startup.
|
||||
let err = config.validate().unwrap_err().to_string();
|
||||
assert!(err.contains("--ssl-ca-certs is required"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_passes_tls_into_config() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_keyfile":"/tmp/key.pem"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
let config = args.into_config();
|
||||
let tls = config.tls.expect("tls configured");
|
||||
assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
|
||||
assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_rejects_out_of_range_cert_reqs() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_cert_reqs":5}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
// The JSON path bypasses clap's range check, so validate() is the only guard.
|
||||
let config = args.into_config();
|
||||
let err = config.validate().unwrap_err().to_string();
|
||||
assert!(err.contains("--ssl-cert-reqs"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_passes_enable_request_id_headers_into_config() {
|
||||
let cli = Cli::try_parse_from([
|
||||
@@ -481,13 +621,13 @@ fn serve_args_reject_unsupported_flag_arg() {
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--ssl-keyfile",
|
||||
"/tmp/key.pem",
|
||||
"--root-path",
|
||||
"/prefix",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '/tmp/key.pem' for '--ssl-keyfile <SSL_KEYFILE>': argument is not implemented in Rust frontend yet
|
||||
error: invalid value '/prefix' for '--root-path <ROOT_PATH>': argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
@@ -562,6 +702,7 @@ fn frontend_args_accept_json() {
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
@@ -586,6 +727,11 @@ fn frontend_args_accept_json() {
|
||||
],
|
||||
),
|
||||
allow_credentials: false,
|
||||
ssl_keyfile: None,
|
||||
ssl_certfile: None,
|
||||
ssl_ca_certs: None,
|
||||
ssl_cert_reqs: 0,
|
||||
ssl_ciphers: None,
|
||||
},
|
||||
},
|
||||
),
|
||||
@@ -798,14 +944,14 @@ fn frontend_args_json_rejects_unsupported_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- ssl_keyfile
|
||||
- root_path
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
@@ -825,16 +971,16 @@ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
let actual = error.to_string().replace(": \n", ":\n");
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- response_role
|
||||
- ssl_keyfile
|
||||
- root_path
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
@@ -1077,6 +1223,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
@@ -1101,6 +1248,11 @@ fn serve_args_accept_handshake_aliases() {
|
||||
],
|
||||
),
|
||||
allow_credentials: false,
|
||||
ssl_keyfile: None,
|
||||
ssl_certfile: None,
|
||||
ssl_ca_certs: None,
|
||||
ssl_cert_reqs: 0,
|
||||
ssl_ciphers: None,
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "python3",
|
||||
@@ -1234,10 +1386,12 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
],
|
||||
allow_credentials: false,
|
||||
},
|
||||
tls: None,
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
keep_alive_timeout: 5s,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&Config {
|
||||
@@ -1315,10 +1469,12 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
],
|
||||
allow_credentials: false,
|
||||
},
|
||||
tls: None,
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
keep_alive_timeout: 5s,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
@@ -1414,10 +1570,12 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
],
|
||||
allow_credentials: false,
|
||||
},
|
||||
tls: None,
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
keep_alive_timeout: 5s,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
|
||||
@@ -526,18 +526,6 @@ pub struct ServerUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub disable_access_log_for_endpoints: Option<Noop>,
|
||||
|
||||
/// The file path to the SSL key file.
|
||||
#[arg(long)]
|
||||
pub ssl_keyfile: Option<Unsupported>,
|
||||
|
||||
/// The file path to the SSL cert file.
|
||||
#[arg(long)]
|
||||
pub ssl_certfile: Option<Unsupported>,
|
||||
|
||||
/// The CA certificates file.
|
||||
#[arg(long)]
|
||||
pub ssl_ca_certs: Option<Unsupported>,
|
||||
|
||||
/// Refresh SSL Context when SSL certificate files change
|
||||
#[arg(
|
||||
long,
|
||||
@@ -547,15 +535,6 @@ pub struct ServerUnsupportedArgs {
|
||||
)]
|
||||
pub enable_ssl_refresh: Option<Unsupported>,
|
||||
|
||||
/// Whether client certificate is required (see stdlib ssl module's).
|
||||
#[arg(long)]
|
||||
pub ssl_cert_reqs: Option<Unsupported>,
|
||||
|
||||
/// SSL cipher suites for HTTPS (TLS 1.2 and below only).
|
||||
/// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
|
||||
#[arg(long)]
|
||||
pub ssl_ciphers: Option<Unsupported>,
|
||||
|
||||
/// FastAPI root_path when app is behind a path based routing proxy.
|
||||
#[arg(long)]
|
||||
pub root_path: Option<Unsupported>,
|
||||
|
||||
Reference in New Issue
Block a user