diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 583a5fb936..20d2b81cc1 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -57,6 +57,7 @@ impl HfChatBackend { processor_config: files.processor_config_path.as_deref(), }, tokenizer.clone(), + options.limit_mm_per_prompt.clone(), )? }; let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref()); @@ -231,6 +232,7 @@ mod tests { chat_template_content_format: Default::default(), chat_template: None, default_chat_template_kwargs: HashMap::new(), + limit_mm_per_prompt: HashMap::new(), }, test_tokenizer(), ) diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index 68f2f6a568..0a0f757d65 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -8,7 +8,7 @@ use serde_json::Value; use vllm_text::{DynTextBackend, TextBackend}; use crate::error::Result; -use crate::multimodal::MultimodalModelInfo; +use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo}; use crate::output::DynChatOutputProcessor; use crate::renderer::DynChatRenderer; use crate::request::ChatRequest; @@ -74,6 +74,9 @@ pub struct LoadModelBackendsOptions { /// Optional server-default keyword arguments merged into every /// chat-template render before request-level `chat_template_kwargs`. pub default_chat_template_kwargs: HashMap, + /// Maximum number of input items allowed per prompt for each modality. + /// Unspecified modalities are unlimited. + pub limit_mm_per_prompt: MmLimitPerPrompt, } /// Shared backends loaded from a model id. diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index e09c3734d9..b032e9bf24 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -23,6 +23,8 @@ pub enum Error { UnsupportedMultimodalContent(&'static str), #[error("`{modality}` input is not supported by this model")] UnsupportedModality { modality: String }, + #[error("At most {limit} {modality}(s) may be provided in one prompt.")] + MmLimitExceeded { modality: String, limit: usize }, #[error("multimodal preprocessing error: {0}")] Multimodal(#[message] String), #[error("{kind} parsing is not available for model `{model_id}`")] @@ -87,7 +89,8 @@ impl Error { Self::Text(error) => error.is_request_validation_error(), Self::UnsupportedMultimodalRenderer | Self::UnsupportedMultimodalContent(_) - | Self::UnsupportedModality { .. } => true, + | Self::UnsupportedModality { .. } + | Self::MmLimitExceeded { .. } => true, _ => false, } diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 9c4f57c041..c0600a6329 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -11,7 +11,7 @@ //! Raw media stays above `vllm-text`; this module lowers it into token IDs and //! opaque tensor payloads before the request is handed to text generation. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs; use std::path::Path; use std::sync::{Arc, LazyLock}; @@ -24,6 +24,7 @@ use llm_multimodal::{ PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry, }; +use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport as _; use tracing::warn; use vllm_engine_core_client::protocol::dtype::ModelDtype; @@ -52,6 +53,71 @@ pub struct MultimodalModelInfo { video: Option, audio: Option, media_connector: Arc, + /// Maximum number of input items allowed per prompt for each modality. + limit_mm_per_prompt: MmLimitPerPrompt, +} + +/// Per-modality item-count limits configured by `--limit-mm-per-prompt`. +/// +/// Modalities absent from the map are unlimited. +pub type MmLimitPerPrompt = HashMap; + +/// Modalities that `--limit-mm-per-prompt` can be keyed by. +/// +/// Closed on purpose: these are exactly the keys Python accepts, per +/// `MultiModalDummyOptionsBuiltins` in `vllm/config/multimodal.py`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MmLimitModality { + Image, + Audio, + Video, +} + +impl MmLimitModality { + /// The wire name, matching Python's modality strings. + pub fn as_str(self) -> &'static str { + match self { + Self::Image => "image", + Self::Audio => "audio", + Self::Video => "video", + } + } +} + +/// One modality's limit, in either of the two shapes Python accepts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + untagged, + expecting = "an item count, or an object with an optional `count` field" +)] +pub enum MmLimitSpec { + /// Legacy form: `"image": 16` + Count(usize), + + /// Configurable form: + /// `"video": {"count": 1, "num_frames": 32}` + Options { + /// Absent means unlimited, matching an absent modality. + #[serde(default, skip_serializing_if = "Option::is_none")] + count: Option, + + /// Preserve Python-owned options for forwarding. Never interpreted + /// here: they size the engine's dummy-profiling encoder cache, which + /// has no Rust counterpart. + #[serde(flatten)] + extra: BTreeMap, + }, +} + +impl MmLimitSpec { + /// The configured item count, or `None` when this modality is unlimited. + pub fn count(&self) -> Option { + match self { + Self::Count(count) => Some(*count), + Self::Options { count, .. } => *count, + } + } } /// Model metadata and tokenizer access shared by all multimodal specs. @@ -287,6 +353,7 @@ impl MultimodalModelInfo { model_type: Option, files: MultimodalConfigFiles<'_>, tokenizer: DynTokenizer, + limit_mm_per_prompt: MmLimitPerPrompt, ) -> Result> { let config = match files.config { Some(path) => { @@ -319,7 +386,12 @@ impl MultimodalModelInfo { tokenizer: TokenizerResolver(tokenizer), }; - Self::from_loaded(context, preprocessor_config, video_preprocessor_config) + Self::from_loaded( + context, + preprocessor_config, + video_preprocessor_config, + limit_mm_per_prompt, + ) } /// Resolve multimodal support from an assembled context and parsed @@ -328,6 +400,7 @@ impl MultimodalModelInfo { context: MultimodalModelContext, preprocessor_config: PreProcessorConfig, video_preprocessor_config: PreProcessorConfig, + limit_mm_per_prompt: MmLimitPerPrompt, ) -> Result> { let (image, video) = Self::resolve_vision_lanes( &context, @@ -356,6 +429,7 @@ impl MultimodalModelInfo { video, audio, media_connector, + limit_mm_per_prompt, })) } @@ -567,7 +641,55 @@ fn input_audio_data_url(data: &str, format: Option<&str>) -> Result { Ok(format!("data:{mime_type};base64,{data}")) } +/// The modality a content part counts against, or `None` for plain text. +/// +/// Embedding inputs share their base modality's budget rather than getting one +/// of their own, matching Python's `modality.replace("_embeds", "")` in +/// `vllm/entrypoints/chat_utils.py`. +fn media_part_limit_modality(part: &MediaContentPart) -> Option { + match part { + MediaContentPart::Text { .. } => None, + MediaContentPart::ImageUrl { .. } + | MediaContentPart::ImageData { .. } + | MediaContentPart::ImageEmbeds { .. } => Some(MmLimitModality::Image), + MediaContentPart::AudioUrl { .. } | MediaContentPart::AudioData { .. } => { + Some(MmLimitModality::Audio) + } + MediaContentPart::VideoUrl { .. } | MediaContentPart::VideoData { .. } => { + Some(MmLimitModality::Video) + } + } +} + impl MultimodalModelInfo { + /// Reject requests exceeding `--limit-mm-per-prompt`'s configured + /// per-modality item count, before any fetch/decode work is spent on them. + /// + /// Modalities without a configured count are unlimited. + fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> { + let mut counts: HashMap = HashMap::new(); + for part in media_parts { + if let Some(modality) = media_part_limit_modality(part) { + *counts.entry(modality).or_default() += 1; + } + } + + for (modality, count) in counts { + let Some(limit) = self.limit_mm_per_prompt.get(&modality).and_then(MmLimitSpec::count) + else { + continue; + }; + if count > limit { + return Err(Error::MmLimitExceeded { + modality: modality.as_str().to_string(), + limit, + }); + } + } + + Ok(()) + } + /// Run media fetch, per-modality preprocessing, prompt expansion, and /// feature build. /// @@ -584,8 +706,7 @@ impl MultimodalModelInfo { if media_parts_len == 0 { return Ok(Vec::new()); } - // TODO: enforce per-modality item-count limits, aligned with the - // engine's `--limit-mm-per-prompt` semantics. + self.validate_mm_limits(&media_parts)?; let fetched = self.fetch_media(media_parts).await?; let mut prepared = Vec::new(); @@ -753,10 +874,11 @@ mod tests { .with_regular_token("<|video_pad|>", QWEN3_VIDEO_PAD_ID) } - fn test_info( + fn test_info_with_limits( model_type: &str, config: serde_json::Value, tokenizer: TestTokenizer, + limit_mm_per_prompt: MmLimitPerPrompt, ) -> MultimodalModelInfo { let context = MultimodalModelContext { model_id: format!("{model_type}-test"), @@ -769,11 +891,20 @@ mod tests { context, PreProcessorConfig::default(), PreProcessorConfig::default(), + limit_mm_per_prompt, ) .unwrap() .unwrap_or_else(|| panic!("{model_type} multimodal support should resolve")) } + fn test_info( + model_type: &str, + config: serde_json::Value, + tokenizer: TestTokenizer, + ) -> MultimodalModelInfo { + test_info_with_limits(model_type, config, tokenizer, HashMap::new()) + } + fn llama4_info() -> MultimodalModelInfo { let config = serde_json::json!({ "model_type": "llama4", @@ -783,16 +914,19 @@ mod tests { test_info("llama4", config, llama4_tokenizer()) } - pub(super) fn qwen3_vl_info() -> MultimodalModelInfo { - let config = serde_json::json!({ + fn qwen3_vl_config() -> serde_json::Value { + serde_json::json!({ "model_type": "qwen3_vl", "image_token_id": QWEN3_IMAGE_PAD_ID, "video_token_id": QWEN3_VIDEO_PAD_ID, "vision_start_token_id": 151652, "vision_end_token_id": 151653, "vision_config": {"patch_size": 16} - }); - test_info("qwen3_vl", config, qwen3_vl_tokenizer()) + }) + } + + pub(super) fn qwen3_vl_info() -> MultimodalModelInfo { + test_info("qwen3_vl", qwen3_vl_config(), qwen3_vl_tokenizer()) } #[test] @@ -853,4 +987,123 @@ mod tests { ); assert!(input_audio_data_url("AAAA", Some("flac")).is_err()); } + + fn image_url_part() -> MediaContentPart { + MediaContentPart::ImageUrl { + url: "https://example.com/image.png".to_string(), + detail: None, + uuid: None, + } + } + + fn qwen3_vl_info_with_limits(limit_mm_per_prompt: MmLimitPerPrompt) -> MultimodalModelInfo { + test_info_with_limits( + "qwen3_vl", + qwen3_vl_config(), + qwen3_vl_tokenizer(), + limit_mm_per_prompt, + ) + } + + #[test] + fn validate_mm_limits_ignores_text_parts() { + let info = qwen3_vl_info(); + let parts = vec![ + MediaContentPart::Text { + text: "hello".to_string(), + }, + MediaContentPart::Text { + text: "world".to_string(), + }, + ]; + assert!(info.validate_mm_limits(&parts).is_ok()); + } + + #[test] + fn validate_mm_limits_leaves_unconfigured_modalities_unlimited() { + let info = qwen3_vl_info(); + let parts: Vec<_> = std::iter::repeat_with(image_url_part).take(1_000).collect(); + + assert!(info.validate_mm_limits(&parts).is_ok()); + } + + #[test] + fn validate_mm_limits_enforces_configured_limit_at_the_boundary() { + let info = qwen3_vl_info_with_limits(HashMap::from([( + MmLimitModality::Image, + MmLimitSpec::Count(1), + )])); + assert!(info.validate_mm_limits(&[image_url_part()]).is_ok()); + + let error = info.validate_mm_limits(&[image_url_part(), image_url_part()]).unwrap_err(); + assert_eq!( + error.to_report_string(), + "At most 1 image(s) may be provided in one prompt." + ); + // Confirms the HTTP-mapping bug found during implementation stays fixed: + // this must map to 400, not 500. + assert!(error.is_request_validation_error()); + } + + #[test] + fn validate_mm_limits_counts_image_embeds_against_the_image_limit() { + let info = qwen3_vl_info_with_limits(HashMap::from([( + MmLimitModality::Image, + MmLimitSpec::Count(1), + )])); + let image_embeds_part = MediaContentPart::ImageEmbeds { + payload: serde_json::Value::String("AAAA".to_string()), + uuid: None, + }; + + let error = info.validate_mm_limits(&[image_url_part(), image_embeds_part]).unwrap_err(); + assert_eq!( + error.to_report_string(), + "At most 1 image(s) may be provided in one prompt." + ); + } + + /// An options object without a `count` carries only profiling keys, which + /// say nothing about how many items are allowed. + #[test] + fn validate_mm_limits_treats_a_count_less_options_object_as_unlimited() { + let info = qwen3_vl_info_with_limits(HashMap::from([( + MmLimitModality::Image, + MmLimitSpec::Options { + count: None, + extra: BTreeMap::from([("width".to_string(), serde_json::json!(512))]), + }, + )])); + + assert!(info.validate_mm_limits(&[image_url_part(), image_url_part()]).is_ok()); + } + + fn parse_limits(json: &str) -> MmLimitPerPrompt { + serde_json::from_str(json).expect("limit map should parse") + } + + #[test] + fn limit_map_parses_both_shapes_python_accepts() { + let limits = parse_limits(r#"{"image": 16, "video": {"count": 1, "num_frames": 32}}"#); + + assert_eq!(limits[&MmLimitModality::Image].count(), Some(16)); + assert_eq!(limits[&MmLimitModality::Video].count(), Some(1)); + assert_eq!(limits.get(&MmLimitModality::Audio), None); + } + + #[test] + fn limit_map_rejects_keys_python_does_not_accept() { + assert!(serde_json::from_str::(r#"{"image_embeds": 1}"#).is_err()); + } + + /// Managed mode forwards this map back to Python as JSON, where + /// `BaseDummyOptions.count` is a non-optional `int` under `extra="forbid"`. + /// Emitting `"count": null` would make the engine subprocess fail to start. + #[test] + fn limit_map_round_trips_without_emitting_a_null_count() { + let source = r#"{"video":{"num_frames":32}}"#; + let encoded = serde_json::to_string(&parse_limits(source)).expect("map should serialize"); + + assert_eq!(encoded, source); + } } diff --git a/rust/src/chat/src/multimodal/audio.rs b/rust/src/chat/src/multimodal/audio.rs index 4910f235ee..41ea47b484 100644 --- a/rust/src/chat/src/multimodal/audio.rs +++ b/rust/src/chat/src/multimodal/audio.rs @@ -107,6 +107,7 @@ mod tests { context, PreProcessorConfig::default(), PreProcessorConfig::default(), + HashMap::new(), ) .unwrap() .expect("Inkling multimodal support") @@ -126,6 +127,7 @@ mod tests { context, PreProcessorConfig::default(), PreProcessorConfig::default(), + HashMap::new(), ) .unwrap() .expect("Qwen3-ASR multimodal support") diff --git a/rust/src/chat/src/multimodal/video.rs b/rust/src/chat/src/multimodal/video.rs index 6c9b961e7c..c2a25b27bd 100644 --- a/rust/src/chat/src/multimodal/video.rs +++ b/rust/src/chat/src/multimodal/video.rs @@ -207,6 +207,7 @@ mod tests { Some("qwen3_vl".to_string()), files, Arc::new(qwen3_vl_tokenizer()), + std::collections::HashMap::new(), ) }; diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 3ae827d286..1572e20467 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -23,6 +23,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as}; use thiserror_ext::AsReport as _; use uuid::Uuid; use vllm_chat::ReasoningParserFactory; +use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_engine_core_client::TransportMode; use vllm_managed_engine::ManagedEngineConfig; use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args}; @@ -186,6 +187,17 @@ pub struct SharedRuntimeArgs { #[serde(default)] pub default_chat_template_kwargs: Option>, + /// The maximum number of input items allowed per prompt for each + /// modality, as a JSON object (e.g. `{"image": 16, "video": 2}`). + /// + /// Also accepts the engine's configurable form + /// (e.g. `{"video": {"count": 1, "num_frames": 32}}`); the extra + /// profiling options are forwarded to the engine untouched. + /// Unspecified modalities are unlimited. + #[arg(long, value_parser = parse_json::, value_name = "JSON", default_value = "{}")] + #[serde(default)] + pub limit_mm_per_prompt: MmLimitPerPrompt, + /// The format to render message content within a chat template. /// /// * "auto" detects the format from the template @@ -348,6 +360,18 @@ impl SharedRuntimeArgs { .expect("profiler config serialization should not fail") } + /// Return the per-modality limits as JSON for managed Python engine + /// forwarding, or `None` when nothing is configured. + /// + /// Round-tripping the parsed map rather than the raw argument keeps the + /// engine's own profiling options (`num_frames`, `width`, ...) intact. + pub fn limit_mm_per_prompt_json(&self) -> Option { + (!self.limit_mm_per_prompt.is_empty()).then(|| { + serde_json::to_string(&self.limit_mm_per_prompt) + .expect("limit-mm-per-prompt serialization should not fail") + }) + } + /// Apply fallback logic for API key configuration from env variables. fn apply_env_api_key_fallback(&mut self) { if self.api_key.is_empty() @@ -399,6 +423,7 @@ impl SharedRuntimeArgs { language_model_only: self.language_model_only, chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs, + limit_mm_per_prompt: self.limit_mm_per_prompt, chat_template_content_format: self.chat_template_content_format, max_logprobs: self.max_logprobs, api_server_options, @@ -451,6 +476,7 @@ impl SharedRuntimeArgs { language_model_only: self.language_model_only, chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs, + limit_mm_per_prompt: self.limit_mm_per_prompt, chat_template_content_format: self.chat_template_content_format, max_logprobs: self.max_logprobs, api_server_options, @@ -666,6 +692,7 @@ impl ServeArgs { self.runtime.disable_log_stats, self.runtime.shutdown_timeout, handshake_port, + self.runtime.limit_mm_per_prompt_json(), ) } } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index bfece8b609..e586477a89 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -64,6 +64,7 @@ fn serve_args_forward_python_flags_with_separator() { http_timeout_keep_alive: None, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: {}, chat_template_content_format: Auto, enable_log_requests: false, enable_prompt_tokens_details: false, @@ -762,6 +763,7 @@ fn frontend_args_accept_json() { http_timeout_keep_alive: None, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: {}, chat_template_content_format: Auto, enable_log_requests: false, enable_prompt_tokens_details: false, @@ -1117,6 +1119,24 @@ fn frontend_args_json_rejects_malformed_json() { "#]].assert_eq(&error.to_string()); } +#[test] +fn serve_args_reject_unsupported_modality_in_limit_mm_per_prompt() { + let error = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--limit-mm-per-prompt", + r#"{"unsupported_modality": 1}"#, + ]) + .unwrap_err(); + + expect![[r#" + error: invalid value '{"unsupported_modality": 1}' for '--limit-mm-per-prompt ': invalid JSON object: unknown variant `unsupported_modality`, expected one of `image`, `audio`, `video` at line 1 column 23 + + For more information, try '--help'. + "#]].assert_eq(&error.to_string()); +} + #[test] fn serve_args_reject_flags_before_model() { let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"]) @@ -1331,6 +1351,7 @@ fn serve_args_accept_handshake_aliases() { http_timeout_keep_alive: None, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: {}, chat_template_content_format: Auto, enable_log_requests: false, enable_prompt_tokens_details: false, @@ -1474,6 +1495,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { language_model_only: false, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: {}, chat_template_content_format: Auto, max_logprobs: None, api_server_options: ApiServerOptions { @@ -1558,6 +1580,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { language_model_only: false, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: {}, chat_template_content_format: Auto, max_logprobs: None, api_server_options: ApiServerOptions { @@ -1660,6 +1683,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present language_model_only: false, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: {}, chat_template_content_format: Auto, max_logprobs: None, api_server_options: ApiServerOptions { diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index 66e8b2e9ab..a48e61dbf7 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -299,11 +299,6 @@ pub struct EngineUnsupportedArgs { )] pub kv_sharing_fast_prefill: Option, - /// The maximum number of input items and options allowed per - /// prompt for each modality. - #[arg(long)] - pub limit_mm_per_prompt: Option, - /// Additional args passed to process media inputs, keyed by modalities. #[arg(long)] pub media_io_kwargs: Option, diff --git a/rust/src/managed-engine/src/cli.rs b/rust/src/managed-engine/src/cli.rs index 26a6e56e79..eadefced23 100644 --- a/rust/src/managed-engine/src/cli.rs +++ b/rust/src/managed-engine/src/cli.rs @@ -91,6 +91,7 @@ impl ManagedEngineArgs { disable_log_stats: bool, shutdown_timeout: u64, handshake_port: u16, + limit_mm_per_prompt: Option, ) -> ManagedEngineConfig { let mut python_args = self.python_args; // Manually forward some args to the Python engine. @@ -126,6 +127,10 @@ impl ManagedEngineArgs { python_args.push("--data-parallel-size-local".to_string()); python_args.push(data_parallel_size_local.to_string()); } + if let Some(limit_mm_per_prompt) = limit_mm_per_prompt { + python_args.push("--limit-mm-per-prompt".to_string()); + python_args.push(limit_mm_per_prompt); + } ManagedEngineConfig { python: self.python, diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 2c2779569a..3877762630 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::collections::HashMap; use std::time::Duration; use anyhow::{Context, Result, bail}; @@ -70,6 +71,7 @@ async fn main() -> Result<()> { language_model_only: false, chat_template: None, default_chat_template_kwargs: None, + limit_mm_per_prompt: HashMap::new(), chat_template_content_format: ChatTemplateContentFormatOption::Auto, max_logprobs: None, api_server_options: ApiServerOptions::default(), diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 91373c82f2..1fb87f2478 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -10,6 +10,7 @@ use axum::http::{HeaderName, HeaderValue, Method}; use educe::Educe; use serde::Serialize; use serde_json::Value; +use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; @@ -184,6 +185,9 @@ pub struct Config { pub chat_template: Option, /// Server-default keyword arguments merged into every chat-template render. pub default_chat_template_kwargs: Option>, + /// Maximum number of input items allowed per prompt for each modality. + /// Unspecified modalities are unlimited. + pub limit_mm_per_prompt: MmLimitPerPrompt, /// How to serialize `message.content` for chat-template rendering. pub chat_template_content_format: ChatTemplateContentFormatOption, /// Optional maximum number of top log probabilities accepted by the diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 470c424cdb..161c0f8ef4 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -101,6 +101,7 @@ async fn build_state(config: &Config) -> Result> { .default_chat_template_kwargs .clone() .unwrap_or_default(), + limit_mm_per_prompt: config.limit_mm_per_prompt.clone(), }, ) .await diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 1c228f99f1..e4e7fed996 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -577,6 +577,12 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R } fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo { + qwen_multimodal_model_info_with_limits(std::collections::HashMap::new()) +} + +fn qwen_multimodal_model_info_with_limits( + limit_mm_per_prompt: vllm_chat::multimodal::MmLimitPerPrompt, +) -> vllm_chat::multimodal::MultimodalModelInfo { let config_path = std::env::temp_dir().join(format!( "vllm-server-qwen-config-{}.json", uuid::Uuid::new_v4() @@ -594,6 +600,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo { ..Default::default() }, Arc::new(fake_chat_tokenizer()), + limit_mm_per_prompt, ) .expect("load multimodal info") .expect("qwen multimodal info is registered"); @@ -2307,6 +2314,74 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() { assert_eq!(json["choices"][0]["message"]["content"], "hi"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() { + // The request is rejected by `--limit-mm-per-prompt` validation before + // ever reaching the engine, so the mock engine task is never awaited. + let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( + b"engine-openai-mm-limit", + default_stream_output_specs(), + Arc::new(FakeChatBackend::with_multimodal_model_info( + qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([( + vllm_chat::multimodal::MmLimitModality::Image, + vllm_chat::multimodal::MmLimitSpec::Count(1), + )])), + )), + ) + .await; + let app = build_router(Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + ))); + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "stream": false, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "describe "}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" + } + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" + } + } + ] + }] + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["error"]["type"], "invalid_request_error"); + assert_eq!( + json["error"]["message"], + "At most 1 image(s) may be provided in one prompt." + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {