[Rust Frontend] Add CORS support (#45753)
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>
This commit is contained in:
+50
-2
@@ -23,8 +23,8 @@ use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
use vllm_server::{
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
|
||||
ParserSelection, RendererSelection,
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||
HttpListenerMode, ParserSelection, RendererSelection,
|
||||
};
|
||||
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
@@ -84,6 +84,13 @@ pub enum Command {
|
||||
Serve(ServeArgs),
|
||||
}
|
||||
|
||||
/// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
|
||||
/// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole
|
||||
/// value as one item keeps clap from treating the field as a repeated flag.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct JsonStringList(pub Vec<String>);
|
||||
|
||||
/// Runtime arguments shared by the external-engine and managed-engine paths.
|
||||
#[serde_as]
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
|
||||
@@ -220,6 +227,30 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub served_model_name: Vec<String>,
|
||||
|
||||
/// CORS allowed origins as a JSON list. `["*"]` allows any origin.
|
||||
#[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
|
||||
#[serde(default = "default_cors_wildcard")]
|
||||
pub allowed_origins: JsonStringList,
|
||||
|
||||
/// CORS allowed methods as a JSON list. `["*"]` allows the standard set.
|
||||
#[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
|
||||
#[serde(default = "default_cors_wildcard")]
|
||||
pub allowed_methods: JsonStringList,
|
||||
|
||||
/// CORS allowed request headers as a JSON list. `["*"]` mirrors the request.
|
||||
#[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
|
||||
#[serde(default = "default_cors_wildcard")]
|
||||
pub allowed_headers: JsonStringList,
|
||||
|
||||
/// Allow CORS credentials (cookies, authorization headers).
|
||||
#[arg(
|
||||
long,
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
#[serde(default)]
|
||||
pub allow_credentials: bool,
|
||||
|
||||
/// Unsupported Python vLLM frontend arguments recognized but not yet
|
||||
/// implemented in Rust.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -264,6 +295,7 @@ impl SharedRuntimeArgs {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
let api_server_options = self.api_server_options();
|
||||
let cors = self.cors_config();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::Bootstrapped {
|
||||
@@ -288,6 +320,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
cors,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
@@ -309,6 +342,7 @@ impl SharedRuntimeArgs {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
let api_server_options = self.api_server_options();
|
||||
let cors = self.cors_config();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
@@ -332,6 +366,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
cors,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
@@ -346,12 +381,25 @@ impl SharedRuntimeArgs {
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
}
|
||||
}
|
||||
|
||||
fn cors_config(&self) -> CorsConfig {
|
||||
CorsConfig {
|
||||
allow_origins: self.allowed_origins.0.clone(),
|
||||
allow_methods: self.allowed_methods.0.clone(),
|
||||
allow_headers: self.allowed_headers.0.clone(),
|
||||
allow_credentials: self.allow_credentials,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_engine_ready_timeout_secs() -> u64 {
|
||||
600
|
||||
}
|
||||
|
||||
fn default_cors_wildcard() -> JsonStringList {
|
||||
JsonStringList(vec!["*".to_string()])
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
||||
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
||||
}
|
||||
|
||||
+165
-11
@@ -49,6 +49,22 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
enable_request_id_headers: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allowed_methods: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allowed_headers: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allow_credentials: false,
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "../vllm/.venv/bin/python",
|
||||
@@ -336,11 +352,17 @@ fn serve_args_reject_unknown_renderer_value() {
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_flag_arg() {
|
||||
let error = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--allow-credentials"])
|
||||
.unwrap_err();
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--ssl-keyfile",
|
||||
"/tmp/key.pem",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'true' for '--allow-credentials [<ALLOW_CREDENTIALS>]': argument is not implemented in Rust frontend yet
|
||||
error: invalid value '/tmp/key.pem' for '--ssl-keyfile <SSL_KEYFILE>': argument is not implemented in Rust frontend yet
|
||||
|
||||
Remove this unsupported argument to continue.
|
||||
|
||||
@@ -348,8 +370,7 @@ fn serve_args_reject_unsupported_flag_arg() {
|
||||
This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
.assert_eq(&error.to_string());
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -423,6 +444,22 @@ fn frontend_args_accept_json() {
|
||||
enable_request_id_headers: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allowed_methods: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allowed_headers: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allow_credentials: false,
|
||||
},
|
||||
},
|
||||
),
|
||||
@@ -558,6 +595,71 @@ fn frontend_args_json_sets_prompt_tokens_details_flag() {
|
||||
assert!(args.runtime.enable_prompt_tokens_details);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_parse_cors_flags() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--allowed-origins",
|
||||
r#"["http://a.com","http://b.com"]"#,
|
||||
"--allowed-methods",
|
||||
r#"["GET","POST"]"#,
|
||||
"--allow-credentials",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(serve) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
serve.runtime.allowed_origins.0,
|
||||
["http://a.com", "http://b.com"]
|
||||
);
|
||||
assert_eq!(serve.runtime.allowed_methods.0, ["GET", "POST"]);
|
||||
assert!(serve.runtime.allow_credentials);
|
||||
// Unspecified lists keep the permissive default.
|
||||
assert_eq!(serve.runtime.allowed_headers.0, ["*"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_cors_defaults_are_permissive() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
|
||||
|
||||
let Command::Serve(serve) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(serve.runtime.allowed_origins.0, ["*"]);
|
||||
assert_eq!(serve.runtime.allowed_methods.0, ["*"]);
|
||||
assert_eq!(serve.runtime.allowed_headers.0, ["*"]);
|
||||
assert!(!serve.runtime.allow_credentials);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_parses_cors_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allowed_origins":["http://a.com"],"allow_credentials":true}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.allowed_origins.0, ["http://a.com"]);
|
||||
assert!(args.runtime.allow_credentials);
|
||||
// Unspecified lists fall back to the permissive default via serde.
|
||||
assert_eq!(args.runtime.allowed_methods.0, ["*"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_rejects_unsupported_fields() {
|
||||
let error = Cli::try_parse_from([
|
||||
@@ -570,14 +672,14 @@ fn frontend_args_json_rejects_unsupported_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}' for '--args-json <JSON>':
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- allow_credentials
|
||||
- ssl_keyfile
|
||||
|
||||
Remove these arguments to continue.
|
||||
|
||||
@@ -597,15 +699,15 @@ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"ssl_keyfile":"/tmp/key.pem"}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
let actual = error.to_string().replace(": \n", ":\n");
|
||||
expect![[r#"
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
|
||||
error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
|
||||
The following arguments are not implemented in Rust frontend yet:
|
||||
- allow_credentials
|
||||
- response_role
|
||||
- ssl_keyfile
|
||||
|
||||
Remove these arguments to continue.
|
||||
@@ -830,6 +932,22 @@ fn serve_args_accept_handshake_aliases() {
|
||||
enable_request_id_headers: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allowed_methods: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allowed_headers: JsonStringList(
|
||||
[
|
||||
"*",
|
||||
],
|
||||
),
|
||||
allow_credentials: false,
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "python3",
|
||||
@@ -951,6 +1069,18 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
"*",
|
||||
],
|
||||
allow_methods: [
|
||||
"*",
|
||||
],
|
||||
allow_headers: [
|
||||
"*",
|
||||
],
|
||||
allow_credentials: false,
|
||||
},
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
@@ -1020,6 +1150,18 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
"*",
|
||||
],
|
||||
allow_methods: [
|
||||
"*",
|
||||
],
|
||||
allow_headers: [
|
||||
"*",
|
||||
],
|
||||
allow_credentials: false,
|
||||
},
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
@@ -1104,6 +1246,18 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
"*",
|
||||
],
|
||||
allow_methods: [
|
||||
"*",
|
||||
],
|
||||
allow_headers: [
|
||||
"*",
|
||||
],
|
||||
allow_credentials: false,
|
||||
},
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
|
||||
@@ -526,27 +526,6 @@ pub struct ServerUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub disable_access_log_for_endpoints: Option<Noop>,
|
||||
|
||||
/// Allow credentials.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-allow-credentials",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub allow_credentials: Option<Unsupported>,
|
||||
|
||||
/// Allowed origins.
|
||||
#[arg(long)]
|
||||
pub allowed_origins: Option<Unsupported>,
|
||||
|
||||
/// Allowed methods.
|
||||
#[arg(long)]
|
||||
pub allowed_methods: Option<Unsupported>,
|
||||
|
||||
/// Allowed headers.
|
||||
#[arg(long)]
|
||||
pub allowed_headers: Option<Unsupported>,
|
||||
|
||||
/// The file path to the SSL key file.
|
||||
#[arg(long)]
|
||||
pub ssl_keyfile: Option<Unsupported>,
|
||||
|
||||
Reference in New Issue
Block a user