[Rust Frontend] Add support for TLS in render server (#54999)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Richard <noreply@moonshot.cn> Signed-off-by: Wen Zhou <wenzhou@redhat.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Richard
parent
6fdee17f84
commit
c7e6e36fa9
@@ -90,6 +90,27 @@ It exposes `/v1/chat/completions/render` and `/v1/completions/render`. Only
|
|||||||
tokenizer and model configuration files are loaded; model weights, PyTorch,
|
tokenizer and model configuration files are loaded; model weights, PyTorch,
|
||||||
and vLLM kernels are not required.
|
and vLLM kernels are not required.
|
||||||
|
|
||||||
|
To serve HTTPS, pass `--ssl-certfile`. If the certificate and private key are
|
||||||
|
in separate files, pass `--ssl-keyfile` for the key; otherwise the key is read from
|
||||||
|
the certificate file. For mTLS, pass `--ssl-ca-certs` with a CA
|
||||||
|
bundle and set `--ssl-cert-reqs` to `1` (optional) or `2` (required) to
|
||||||
|
verify client certificates. Use `--ssl-ciphers` to override the default
|
||||||
|
OpenSSL cipher list.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Combined PEM (cert + key in one file):
|
||||||
|
vllm-rs \
|
||||||
|
render Qwen/Qwen3-32B \
|
||||||
|
--host 127.0.0.1 --max-model-len 32768 \
|
||||||
|
--ssl-certfile /path/to/combined.pem
|
||||||
|
|
||||||
|
# Separate cert and key files:
|
||||||
|
vllm-rs \
|
||||||
|
render Qwen/Qwen3-32B \
|
||||||
|
--host 127.0.0.1 --max-model-len 32768 \
|
||||||
|
--ssl-certfile /path/to/cert.pem --ssl-keyfile /path/to/key.pem
|
||||||
|
```
|
||||||
|
|
||||||
The render endpoints return the public token-in `GenerateRequest` consumed by
|
The render endpoints return the public token-in `GenerateRequest` consumed by
|
||||||
the Rust `/inference/v1/generate` endpoint. A chat render response, or one item
|
the Rust `/inference/v1/generate` endpoint. A chat render response, or one item
|
||||||
from a completion render response, can be submitted to that endpoint without
|
from a completion render response, can be submitted to that endpoint without
|
||||||
|
|||||||
+15
-47
@@ -7,6 +7,7 @@
|
|||||||
//! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
|
//! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
|
||||||
//! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
|
//! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
|
||||||
|
|
||||||
|
mod ssl;
|
||||||
mod unsupported;
|
mod unsupported;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -30,9 +31,10 @@ use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_arg
|
|||||||
use vllm_server::{
|
use vllm_server::{
|
||||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
|
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
|
||||||
RendererSelection, TlsConfig,
|
RendererSelection,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use crate::cli::ssl::SslArgs;
|
||||||
use crate::cli::unsupported::UnsupportedArgs;
|
use crate::cli::unsupported::UnsupportedArgs;
|
||||||
|
|
||||||
/// Top-level parser for the `vllm-rs` binary.
|
/// Top-level parser for the `vllm-rs` binary.
|
||||||
@@ -143,10 +145,15 @@ pub struct RenderArgs {
|
|||||||
/// Maximum accepted logprobs count; -1 disables the cap.
|
/// Maximum accepted logprobs count; -1 disables the cap.
|
||||||
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
||||||
max_logprobs: Option<i32>,
|
max_logprobs: Option<i32>,
|
||||||
|
/// TLS options for HTTPS/mTLS.
|
||||||
|
#[command(flatten)]
|
||||||
|
ssl: SslArgs,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RenderArgs {
|
impl RenderArgs {
|
||||||
pub(super) fn into_config(self) -> RenderConfig {
|
pub(super) fn into_config(self) -> RenderConfig {
|
||||||
|
let tls = self.ssl.tls_config();
|
||||||
|
|
||||||
RenderConfig {
|
RenderConfig {
|
||||||
model: self.model,
|
model: self.model,
|
||||||
served_model_name: self.served_model_name,
|
served_model_name: self.served_model_name,
|
||||||
@@ -160,6 +167,7 @@ impl RenderArgs {
|
|||||||
chat_template_content_format: self.chat_template_content_format,
|
chat_template_content_format: self.chat_template_content_format,
|
||||||
max_model_len: self.max_model_len,
|
max_model_len: self.max_model_len,
|
||||||
max_logprobs: self.max_logprobs,
|
max_logprobs: self.max_logprobs,
|
||||||
|
tls,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -360,33 +368,10 @@ pub struct SharedRuntimeArgs {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub allow_credentials: bool,
|
pub allow_credentials: bool,
|
||||||
|
|
||||||
/// The file path to the SSL key file. When omitted, the key is read from
|
/// TLS options for HTTPS/mTLS.
|
||||||
/// `--ssl-certfile` (combined PEM).
|
#[command(flatten)]
|
||||||
#[arg(long)]
|
#[serde(default, flatten)]
|
||||||
#[serde(default)]
|
pub ssl: SslArgs,
|
||||||
pub ssl_keyfile: Option<String>,
|
|
||||||
|
|
||||||
/// The file path to the SSL cert file. Enables TLS when set.
|
|
||||||
#[arg(long)]
|
|
||||||
#[serde(default)]
|
|
||||||
pub ssl_certfile: Option<String>,
|
|
||||||
|
|
||||||
/// The CA certificates file used to verify client certificates (mTLS).
|
|
||||||
#[arg(long)]
|
|
||||||
#[serde(default)]
|
|
||||||
pub ssl_ca_certs: Option<String>,
|
|
||||||
|
|
||||||
/// Whether a client certificate is required: 0 = none, 1 = optional,
|
|
||||||
/// 2 = required (mirrors Python's `ssl.CERT_*`).
|
|
||||||
#[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
|
|
||||||
#[serde(default)]
|
|
||||||
pub ssl_cert_reqs: i32,
|
|
||||||
|
|
||||||
/// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
|
|
||||||
/// When unset, the linked OpenSSL's default suites are used.
|
|
||||||
#[arg(long)]
|
|
||||||
#[serde(default)]
|
|
||||||
pub ssl_ciphers: Option<String>,
|
|
||||||
|
|
||||||
/// Profiler configuration forwarded by the Python supervisor.
|
/// Profiler configuration forwarded by the Python supervisor.
|
||||||
///
|
///
|
||||||
@@ -478,7 +463,7 @@ impl SharedRuntimeArgs {
|
|||||||
let keep_alive_timeout = self.keep_alive_timeout();
|
let keep_alive_timeout = self.keep_alive_timeout();
|
||||||
let api_server_options = self.api_server_options();
|
let api_server_options = self.api_server_options();
|
||||||
let cors = self.cors_config();
|
let cors = self.cors_config();
|
||||||
let tls = self.tls_config();
|
let tls = self.ssl.tls_config();
|
||||||
let profiler = self.profiler();
|
let profiler = self.profiler();
|
||||||
|
|
||||||
Config {
|
Config {
|
||||||
@@ -536,7 +521,7 @@ impl SharedRuntimeArgs {
|
|||||||
let keep_alive_timeout = self.keep_alive_timeout();
|
let keep_alive_timeout = self.keep_alive_timeout();
|
||||||
let api_server_options = self.api_server_options();
|
let api_server_options = self.api_server_options();
|
||||||
let cors = self.cors_config();
|
let cors = self.cors_config();
|
||||||
let tls = self.tls_config();
|
let tls = self.ssl.tls_config();
|
||||||
let profiler = self.profiler();
|
let profiler = self.profiler();
|
||||||
|
|
||||||
Config {
|
Config {
|
||||||
@@ -591,23 +576,6 @@ impl SharedRuntimeArgs {
|
|||||||
allow_credentials: self.allow_credentials,
|
allow_credentials: self.allow_credentials,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the TLS config: `Some` when any `ssl_*` argument is set, else
|
|
||||||
/// `None` (plaintext). The combination is validated in [`Config::validate`].
|
|
||||||
fn tls_config(&self) -> Option<TlsConfig> {
|
|
||||||
let tls_requested = self.ssl_certfile.is_some()
|
|
||||||
|| self.ssl_keyfile.is_some()
|
|
||||||
|| self.ssl_ca_certs.is_some()
|
|
||||||
|| self.ssl_cert_reqs != 0
|
|
||||||
|| self.ssl_ciphers.is_some();
|
|
||||||
tls_requested.then(|| TlsConfig {
|
|
||||||
cert_file: self.ssl_certfile.clone(),
|
|
||||||
key_file: self.ssl_keyfile.clone(),
|
|
||||||
ca_certs: self.ssl_ca_certs.clone(),
|
|
||||||
cert_reqs: self.ssl_cert_reqs,
|
|
||||||
ciphers: self.ssl_ciphers.clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_engine_ready_timeout_secs() -> u64 {
|
fn default_engine_ready_timeout_secs() -> u64 {
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
//! TLS CLI options mirroring Python's uvicorn `ssl_*` arguments, shared by the
|
||||||
|
//! OpenAI frontend and the render server.
|
||||||
|
|
||||||
|
use clap::Args;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use vllm_server::TlsConfig;
|
||||||
|
|
||||||
|
/// TLS options mirroring Python's uvicorn `ssl_*` arguments, shared by the
|
||||||
|
/// OpenAI frontend and the render server.
|
||||||
|
#[derive(Clone, Debug, Default, Args, PartialEq, Eq, Deserialize)]
|
||||||
|
pub struct SslArgs {
|
||||||
|
/// The file path to the SSL key file. When omitted, the key is read from
|
||||||
|
/// `--ssl-certfile` (combined PEM).
|
||||||
|
#[arg(long)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub ssl_keyfile: Option<String>,
|
||||||
|
|
||||||
|
/// The file path to the SSL cert file. Enables TLS when set.
|
||||||
|
#[arg(long)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub ssl_certfile: Option<String>,
|
||||||
|
|
||||||
|
/// The CA certificates file used to verify client certificates (mTLS).
|
||||||
|
#[arg(long)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub ssl_ca_certs: Option<String>,
|
||||||
|
|
||||||
|
/// Whether a client certificate is required: 0 = none, 1 = optional,
|
||||||
|
/// 2 = required (mirrors Python's `ssl.CERT_*`).
|
||||||
|
#[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
|
||||||
|
#[serde(default)]
|
||||||
|
pub ssl_cert_reqs: i32,
|
||||||
|
|
||||||
|
/// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
|
||||||
|
/// When unset, the linked OpenSSL's default suites are used.
|
||||||
|
#[arg(long)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub ssl_ciphers: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SslArgs {
|
||||||
|
/// Build the TLS config: `Some` when any `ssl_*` argument is set, else
|
||||||
|
/// `None` (plaintext). The combination is validated in [`TlsConfig::validate`].
|
||||||
|
pub fn tls_config(&self) -> Option<TlsConfig> {
|
||||||
|
let tls_requested = self.ssl_certfile.is_some()
|
||||||
|
|| self.ssl_keyfile.is_some()
|
||||||
|
|| self.ssl_ca_certs.is_some()
|
||||||
|
|| self.ssl_cert_reqs != 0
|
||||||
|
|| self.ssl_ciphers.is_some();
|
||||||
|
tls_requested.then(|| TlsConfig {
|
||||||
|
cert_file: self.ssl_certfile.clone(),
|
||||||
|
key_file: self.ssl_keyfile.clone(),
|
||||||
|
ca_certs: self.ssl_ca_certs.clone(),
|
||||||
|
cert_reqs: self.ssl_cert_reqs,
|
||||||
|
ciphers: self.ssl_ciphers.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -31,7 +31,7 @@ fn bench_serve_args_parse_without_managed_engine_repartition() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_args_build_supported_config() {
|
fn render_args_build_config_without_tls() {
|
||||||
let cli = Cli::try_parse_from([
|
let cli = Cli::try_parse_from([
|
||||||
"vllm-rs",
|
"vllm-rs",
|
||||||
"render",
|
"render",
|
||||||
@@ -63,6 +63,76 @@ fn render_args_build_supported_config() {
|
|||||||
assert_eq!(config.reasoning_parser, ParserSelection::Auto);
|
assert_eq!(config.reasoning_parser, ParserSelection::Auto);
|
||||||
assert_eq!(config.renderer, RendererSelection::DeepSeekV32);
|
assert_eq!(config.renderer, RendererSelection::DeepSeekV32);
|
||||||
assert_eq!(config.max_logprobs, Some(-1));
|
assert_eq!(config.max_logprobs, Some(-1));
|
||||||
|
assert!(config.tls.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn render_args_build_config_with_tls() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"render",
|
||||||
|
"Qwen/Qwen2.5-0.5B-Instruct",
|
||||||
|
"--max-model-len",
|
||||||
|
"32768",
|
||||||
|
"--ssl-certfile",
|
||||||
|
"/tmp/cert.pem",
|
||||||
|
"--ssl-keyfile",
|
||||||
|
"/tmp/key.pem",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Render(args) = cli.command else {
|
||||||
|
panic!("expected render args");
|
||||||
|
};
|
||||||
|
let config = args.into_config();
|
||||||
|
|
||||||
|
let tls = config.tls.expect("TLS should be enabled");
|
||||||
|
assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
|
||||||
|
assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn render_args_reject_tls_without_certificate() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"render",
|
||||||
|
"Qwen/Qwen2.5-0.5B-Instruct",
|
||||||
|
"--max-model-len",
|
||||||
|
"32768",
|
||||||
|
"--ssl-keyfile",
|
||||||
|
"/tmp/key.pem",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Render(args) = cli.command else {
|
||||||
|
panic!("expected render args");
|
||||||
|
};
|
||||||
|
let error = args.into_config().validate().unwrap_err();
|
||||||
|
|
||||||
|
assert!(error.to_string().contains("--ssl-certfile is required"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn render_args_reject_client_cert_verification_without_ca() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"render",
|
||||||
|
"Qwen/Qwen2.5-0.5B-Instruct",
|
||||||
|
"--max-model-len",
|
||||||
|
"32768",
|
||||||
|
"--ssl-certfile",
|
||||||
|
"/tmp/cert.pem",
|
||||||
|
"--ssl-cert-reqs",
|
||||||
|
"2",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Render(args) = cli.command else {
|
||||||
|
panic!("expected render args");
|
||||||
|
};
|
||||||
|
let error = args.into_config().validate().unwrap_err();
|
||||||
|
|
||||||
|
assert!(error.to_string().contains("--ssl-ca-certs is required"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -127,11 +197,13 @@ fn serve_args_forward_python_flags_with_separator() {
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
allow_credentials: false,
|
allow_credentials: false,
|
||||||
ssl_keyfile: None,
|
ssl: SslArgs {
|
||||||
ssl_certfile: None,
|
ssl_keyfile: None,
|
||||||
ssl_ca_certs: None,
|
ssl_certfile: None,
|
||||||
ssl_cert_reqs: 0,
|
ssl_ca_certs: None,
|
||||||
ssl_ciphers: None,
|
ssl_cert_reqs: 0,
|
||||||
|
ssl_ciphers: None,
|
||||||
|
},
|
||||||
profiler_config: None,
|
profiler_config: None,
|
||||||
},
|
},
|
||||||
managed_engine: ManagedEngineArgs {
|
managed_engine: ManagedEngineArgs {
|
||||||
@@ -944,11 +1016,13 @@ fn frontend_args_accept_json() {
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
allow_credentials: false,
|
allow_credentials: false,
|
||||||
ssl_keyfile: None,
|
ssl: SslArgs {
|
||||||
ssl_certfile: None,
|
ssl_keyfile: None,
|
||||||
ssl_ca_certs: None,
|
ssl_certfile: None,
|
||||||
ssl_cert_reqs: 0,
|
ssl_ca_certs: None,
|
||||||
ssl_ciphers: None,
|
ssl_cert_reqs: 0,
|
||||||
|
ssl_ciphers: None,
|
||||||
|
},
|
||||||
profiler_config: None,
|
profiler_config: None,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -1539,11 +1613,13 @@ fn serve_args_accept_handshake_aliases() {
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
allow_credentials: false,
|
allow_credentials: false,
|
||||||
ssl_keyfile: None,
|
ssl: SslArgs {
|
||||||
ssl_certfile: None,
|
ssl_keyfile: None,
|
||||||
ssl_ca_certs: None,
|
ssl_certfile: None,
|
||||||
ssl_cert_reqs: 0,
|
ssl_ca_certs: None,
|
||||||
ssl_ciphers: None,
|
ssl_cert_reqs: 0,
|
||||||
|
ssl_ciphers: None,
|
||||||
|
},
|
||||||
profiler_config: None,
|
profiler_config: None,
|
||||||
},
|
},
|
||||||
managed_engine: ManagedEngineArgs {
|
managed_engine: ManagedEngineArgs {
|
||||||
|
|||||||
@@ -14,7 +14,11 @@ use vllm_chat::{
|
|||||||
};
|
};
|
||||||
use vllm_text::TextRequestProcessor;
|
use vllm_text::TextRequestProcessor;
|
||||||
|
|
||||||
use crate::{HttpListenerMode, listener::Listener};
|
use crate::{
|
||||||
|
HttpListenerMode, TlsConfig,
|
||||||
|
listener::{Listener, MaybeTlsListener},
|
||||||
|
tls,
|
||||||
|
};
|
||||||
|
|
||||||
/// Configuration for the engine-free text rendering server.
|
/// Configuration for the engine-free text rendering server.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -31,6 +35,7 @@ pub struct RenderConfig {
|
|||||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||||
pub max_model_len: u32,
|
pub max_model_len: u32,
|
||||||
pub max_logprobs: Option<i32>,
|
pub max_logprobs: Option<i32>,
|
||||||
|
pub tls: Option<TlsConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RenderConfig {
|
impl RenderConfig {
|
||||||
@@ -41,6 +46,9 @@ impl RenderConfig {
|
|||||||
if self.max_logprobs.is_some_and(|value| value < -1) {
|
if self.max_logprobs.is_some_and(|value| value < -1) {
|
||||||
bail!("max_logprobs must be non-negative or -1");
|
bail!("max_logprobs must be non-negative or -1");
|
||||||
}
|
}
|
||||||
|
if let Some(tls) = &self.tls {
|
||||||
|
tls.validate()?;
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -87,6 +95,12 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
|||||||
/// to an inference engine.
|
/// to an inference engine.
|
||||||
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
|
pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) -> Result<()> {
|
||||||
config.validate().context("invalid render server configuration")?;
|
config.validate().context("invalid render server configuration")?;
|
||||||
|
let tls_config = config
|
||||||
|
.tls
|
||||||
|
.as_ref()
|
||||||
|
.map(tls::build_server_config)
|
||||||
|
.transpose()
|
||||||
|
.context("invalid TLS configuration")?;
|
||||||
let state = tokio::select! {
|
let state = tokio::select! {
|
||||||
result = build_state(&config) => result?,
|
result = build_state(&config) => result?,
|
||||||
_ = shutdown.cancelled() => return Ok(()),
|
_ = shutdown.cancelled() => return Ok(()),
|
||||||
@@ -99,8 +113,18 @@ pub async fn serve_render(config: RenderConfig, shutdown: CancellationToken) ->
|
|||||||
.await
|
.await
|
||||||
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
|
.with_context(|| format!("failed to bind {}:{}", config.host, config.port))?;
|
||||||
let address = listener.local_addr_display()?;
|
let address = listener.local_addr_display()?;
|
||||||
|
let scheme = if tls_config.is_some() {
|
||||||
|
"https"
|
||||||
|
} else {
|
||||||
|
"http"
|
||||||
|
};
|
||||||
|
let listener = match tls_config {
|
||||||
|
Some(context) => MaybeTlsListener::tls(listener, context),
|
||||||
|
None => MaybeTlsListener::plain(listener),
|
||||||
|
};
|
||||||
info!(
|
info!(
|
||||||
%address,
|
%address,
|
||||||
|
scheme,
|
||||||
model = %config.model,
|
model = %config.model,
|
||||||
"starting engine-free Rust render server"
|
"starting engine-free Rust render server"
|
||||||
);
|
);
|
||||||
@@ -132,6 +156,7 @@ mod tests {
|
|||||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||||
max_model_len: 128,
|
max_model_len: 128,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
|
tls: None,
|
||||||
},
|
},
|
||||||
shutdown,
|
shutdown,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user