[Rust Frontend] Add --limit-mm-per-prompt support (#49604)

Signed-off-by: Maria Guevara <kawaiiplush14@gmail.com>
This commit is contained in:
Maria Guevara
2026-07-29 17:21:41 +08:00
committed by GitHub
parent 5b14019576
commit c44e191b01
14 changed files with 413 additions and 16 deletions
+27
View File
@@ -23,6 +23,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
use thiserror_ext::AsReport as _;
use uuid::Uuid;
use vllm_chat::ReasoningParserFactory;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_engine_core_client::TransportMode;
use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
@@ -186,6 +187,17 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// The maximum number of input items allowed per prompt for each
/// modality, as a JSON object (e.g. `{"image": 16, "video": 2}`).
///
/// Also accepts the engine's configurable form
/// (e.g. `{"video": {"count": 1, "num_frames": 32}}`); the extra
/// profiling options are forwarded to the engine untouched.
/// Unspecified modalities are unlimited.
#[arg(long, value_parser = parse_json::<MmLimitPerPrompt>, value_name = "JSON", default_value = "{}")]
#[serde(default)]
pub limit_mm_per_prompt: MmLimitPerPrompt,
/// The format to render message content within a chat template.
///
/// * "auto" detects the format from the template
@@ -348,6 +360,18 @@ impl SharedRuntimeArgs {
.expect("profiler config serialization should not fail")
}
/// Return the per-modality limits as JSON for managed Python engine
/// forwarding, or `None` when nothing is configured.
///
/// Round-tripping the parsed map rather than the raw argument keeps the
/// engine's own profiling options (`num_frames`, `width`, ...) intact.
pub fn limit_mm_per_prompt_json(&self) -> Option<String> {
(!self.limit_mm_per_prompt.is_empty()).then(|| {
serde_json::to_string(&self.limit_mm_per_prompt)
.expect("limit-mm-per-prompt serialization should not fail")
})
}
/// Apply fallback logic for API key configuration from env variables.
fn apply_env_api_key_fallback(&mut self) {
if self.api_key.is_empty()
@@ -399,6 +423,7 @@ impl SharedRuntimeArgs {
language_model_only: self.language_model_only,
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
limit_mm_per_prompt: self.limit_mm_per_prompt,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
@@ -451,6 +476,7 @@ impl SharedRuntimeArgs {
language_model_only: self.language_model_only,
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
limit_mm_per_prompt: self.limit_mm_per_prompt,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
@@ -666,6 +692,7 @@ impl ServeArgs {
self.runtime.disable_log_stats,
self.runtime.shutdown_timeout,
handshake_port,
self.runtime.limit_mm_per_prompt_json(),
)
}
}
+24
View File
@@ -64,6 +64,7 @@ fn serve_args_forward_python_flags_with_separator() {
http_timeout_keep_alive: None,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -762,6 +763,7 @@ fn frontend_args_accept_json() {
http_timeout_keep_alive: None,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1117,6 +1119,24 @@ fn frontend_args_json_rejects_malformed_json() {
"#]].assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_unsupported_modality_in_limit_mm_per_prompt() {
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--limit-mm-per-prompt",
r#"{"unsupported_modality": 1}"#,
])
.unwrap_err();
expect![[r#"
error: invalid value '{"unsupported_modality": 1}' for '--limit-mm-per-prompt <JSON>': invalid JSON object: unknown variant `unsupported_modality`, expected one of `image`, `audio`, `video` at line 1 column 23
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_flags_before_model() {
let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
@@ -1331,6 +1351,7 @@ fn serve_args_accept_handshake_aliases() {
http_timeout_keep_alive: None,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1474,6 +1495,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
language_model_only: false,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
@@ -1558,6 +1580,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
language_model_only: false,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
@@ -1660,6 +1683,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
language_model_only: false,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: {},
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
-5
View File
@@ -299,11 +299,6 @@ pub struct EngineUnsupportedArgs {
)]
pub kv_sharing_fast_prefill: Option<Unsupported>,
/// The maximum number of input items and options allowed per
/// prompt for each modality.
#[arg(long)]
pub limit_mm_per_prompt: Option<Unsupported>,
/// Additional args passed to process media inputs, keyed by modalities.
#[arg(long)]
pub media_io_kwargs: Option<Unsupported>,