[Rust Frontend] Add --limit-mm-per-prompt support (#49604)

Signed-off-by: Maria Guevara <kawaiiplush14@gmail.com>
This commit is contained in:
Maria Guevara
2026-07-29 17:21:41 +08:00
committed by GitHub
parent 5b14019576
commit c44e191b01
14 changed files with 413 additions and 16 deletions
+2
View File
@@ -57,6 +57,7 @@ impl HfChatBackend {
processor_config: files.processor_config_path.as_deref(),
},
tokenizer.clone(),
options.limit_mm_per_prompt.clone(),
)?
};
let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());
@@ -231,6 +232,7 @@ mod tests {
chat_template_content_format: Default::default(),
chat_template: None,
default_chat_template_kwargs: HashMap::new(),
limit_mm_per_prompt: HashMap::new(),
},
test_tokenizer(),
)
+4 -1
View File
@@ -8,7 +8,7 @@ use serde_json::Value;
use vllm_text::{DynTextBackend, TextBackend};
use crate::error::Result;
use crate::multimodal::MultimodalModelInfo;
use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
use crate::output::DynChatOutputProcessor;
use crate::renderer::DynChatRenderer;
use crate::request::ChatRequest;
@@ -74,6 +74,9 @@ pub struct LoadModelBackendsOptions {
/// Optional server-default keyword arguments merged into every
/// chat-template render before request-level `chat_template_kwargs`.
pub default_chat_template_kwargs: HashMap<String, Value>,
/// Maximum number of input items allowed per prompt for each modality.
/// Unspecified modalities are unlimited.
pub limit_mm_per_prompt: MmLimitPerPrompt,
}
/// Shared backends loaded from a model id.
+4 -1
View File
@@ -23,6 +23,8 @@ pub enum Error {
UnsupportedMultimodalContent(&'static str),
#[error("`{modality}` input is not supported by this model")]
UnsupportedModality { modality: String },
#[error("At most {limit} {modality}(s) may be provided in one prompt.")]
MmLimitExceeded { modality: String, limit: usize },
#[error("multimodal preprocessing error: {0}")]
Multimodal(#[message] String),
#[error("{kind} parsing is not available for model `{model_id}`")]
@@ -87,7 +89,8 @@ impl Error {
Self::Text(error) => error.is_request_validation_error(),
Self::UnsupportedMultimodalRenderer
| Self::UnsupportedMultimodalContent(_)
| Self::UnsupportedModality { .. } => true,
| Self::UnsupportedModality { .. }
| Self::MmLimitExceeded { .. } => true,
_ => false,
}
+262 -9
View File
@@ -11,7 +11,7 @@
//! Raw media stays above `vllm-text`; this module lowers it into token IDs and
//! opaque tensor payloads before the request is handed to text generation.
use std::collections::HashSet;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs;
use std::path::Path;
use std::sync::{Arc, LazyLock};
@@ -24,6 +24,7 @@ use llm_multimodal::{
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
VisionProcessorRegistry,
};
use serde::{Deserialize, Serialize};
use thiserror_ext::AsReport as _;
use tracing::warn;
use vllm_engine_core_client::protocol::dtype::ModelDtype;
@@ -52,6 +53,71 @@ pub struct MultimodalModelInfo {
video: Option<ModalitySupport>,
audio: Option<AudioModalitySupport>,
media_connector: Arc<MediaConnector>,
/// Maximum number of input items allowed per prompt for each modality.
limit_mm_per_prompt: MmLimitPerPrompt,
}
/// Per-modality item-count limits configured by `--limit-mm-per-prompt`.
///
/// Modalities absent from the map are unlimited.
pub type MmLimitPerPrompt = HashMap<MmLimitModality, MmLimitSpec>;
/// Modalities that `--limit-mm-per-prompt` can be keyed by.
///
/// Closed on purpose: these are exactly the keys Python accepts, per
/// `MultiModalDummyOptionsBuiltins` in `vllm/config/multimodal.py`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MmLimitModality {
Image,
Audio,
Video,
}
impl MmLimitModality {
/// The wire name, matching Python's modality strings.
pub fn as_str(self) -> &'static str {
match self {
Self::Image => "image",
Self::Audio => "audio",
Self::Video => "video",
}
}
}
/// One modality's limit, in either of the two shapes Python accepts.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(
untagged,
expecting = "an item count, or an object with an optional `count` field"
)]
pub enum MmLimitSpec {
/// Legacy form: `"image": 16`
Count(usize),
/// Configurable form:
/// `"video": {"count": 1, "num_frames": 32}`
Options {
/// Absent means unlimited, matching an absent modality.
#[serde(default, skip_serializing_if = "Option::is_none")]
count: Option<usize>,
/// Preserve Python-owned options for forwarding. Never interpreted
/// here: they size the engine's dummy-profiling encoder cache, which
/// has no Rust counterpart.
#[serde(flatten)]
extra: BTreeMap<String, serde_json::Value>,
},
}
impl MmLimitSpec {
/// The configured item count, or `None` when this modality is unlimited.
pub fn count(&self) -> Option<usize> {
match self {
Self::Count(count) => Some(*count),
Self::Options { count, .. } => *count,
}
}
}
/// Model metadata and tokenizer access shared by all multimodal specs.
@@ -287,6 +353,7 @@ impl MultimodalModelInfo {
model_type: Option<String>,
files: MultimodalConfigFiles<'_>,
tokenizer: DynTokenizer,
limit_mm_per_prompt: MmLimitPerPrompt,
) -> Result<Option<Self>> {
let config = match files.config {
Some(path) => {
@@ -319,7 +386,12 @@ impl MultimodalModelInfo {
tokenizer: TokenizerResolver(tokenizer),
};
Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
Self::from_loaded(
context,
preprocessor_config,
video_preprocessor_config,
limit_mm_per_prompt,
)
}
/// Resolve multimodal support from an assembled context and parsed
@@ -328,6 +400,7 @@ impl MultimodalModelInfo {
context: MultimodalModelContext,
preprocessor_config: PreProcessorConfig,
video_preprocessor_config: PreProcessorConfig,
limit_mm_per_prompt: MmLimitPerPrompt,
) -> Result<Option<Self>> {
let (image, video) = Self::resolve_vision_lanes(
&context,
@@ -356,6 +429,7 @@ impl MultimodalModelInfo {
video,
audio,
media_connector,
limit_mm_per_prompt,
}))
}
@@ -567,7 +641,55 @@ fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
Ok(format!("data:{mime_type};base64,{data}"))
}
/// The modality a content part counts against, or `None` for plain text.
///
/// Embedding inputs share their base modality's budget rather than getting one
/// of their own, matching Python's `modality.replace("_embeds", "")` in
/// `vllm/entrypoints/chat_utils.py`.
fn media_part_limit_modality(part: &MediaContentPart) -> Option<MmLimitModality> {
match part {
MediaContentPart::Text { .. } => None,
MediaContentPart::ImageUrl { .. }
| MediaContentPart::ImageData { .. }
| MediaContentPart::ImageEmbeds { .. } => Some(MmLimitModality::Image),
MediaContentPart::AudioUrl { .. } | MediaContentPart::AudioData { .. } => {
Some(MmLimitModality::Audio)
}
MediaContentPart::VideoUrl { .. } | MediaContentPart::VideoData { .. } => {
Some(MmLimitModality::Video)
}
}
}
impl MultimodalModelInfo {
/// Reject requests exceeding `--limit-mm-per-prompt`'s configured
/// per-modality item count, before any fetch/decode work is spent on them.
///
/// Modalities without a configured count are unlimited.
fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> {
let mut counts: HashMap<MmLimitModality, usize> = HashMap::new();
for part in media_parts {
if let Some(modality) = media_part_limit_modality(part) {
*counts.entry(modality).or_default() += 1;
}
}
for (modality, count) in counts {
let Some(limit) = self.limit_mm_per_prompt.get(&modality).and_then(MmLimitSpec::count)
else {
continue;
};
if count > limit {
return Err(Error::MmLimitExceeded {
modality: modality.as_str().to_string(),
limit,
});
}
}
Ok(())
}
/// Run media fetch, per-modality preprocessing, prompt expansion, and
/// feature build.
///
@@ -584,8 +706,7 @@ impl MultimodalModelInfo {
if media_parts_len == 0 {
return Ok(Vec::new());
}
// TODO: enforce per-modality item-count limits, aligned with the
// engine's `--limit-mm-per-prompt` semantics.
self.validate_mm_limits(&media_parts)?;
let fetched = self.fetch_media(media_parts).await?;
let mut prepared = Vec::new();
@@ -753,10 +874,11 @@ mod tests {
.with_regular_token("<|video_pad|>", QWEN3_VIDEO_PAD_ID)
}
fn test_info(
fn test_info_with_limits(
model_type: &str,
config: serde_json::Value,
tokenizer: TestTokenizer,
limit_mm_per_prompt: MmLimitPerPrompt,
) -> MultimodalModelInfo {
let context = MultimodalModelContext {
model_id: format!("{model_type}-test"),
@@ -769,11 +891,20 @@ mod tests {
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
limit_mm_per_prompt,
)
.unwrap()
.unwrap_or_else(|| panic!("{model_type} multimodal support should resolve"))
}
fn test_info(
model_type: &str,
config: serde_json::Value,
tokenizer: TestTokenizer,
) -> MultimodalModelInfo {
test_info_with_limits(model_type, config, tokenizer, HashMap::new())
}
fn llama4_info() -> MultimodalModelInfo {
let config = serde_json::json!({
"model_type": "llama4",
@@ -783,16 +914,19 @@ mod tests {
test_info("llama4", config, llama4_tokenizer())
}
pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
let config = serde_json::json!({
fn qwen3_vl_config() -> serde_json::Value {
serde_json::json!({
"model_type": "qwen3_vl",
"image_token_id": QWEN3_IMAGE_PAD_ID,
"video_token_id": QWEN3_VIDEO_PAD_ID,
"vision_start_token_id": 151652,
"vision_end_token_id": 151653,
"vision_config": {"patch_size": 16}
});
test_info("qwen3_vl", config, qwen3_vl_tokenizer())
})
}
pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
test_info("qwen3_vl", qwen3_vl_config(), qwen3_vl_tokenizer())
}
#[test]
@@ -853,4 +987,123 @@ mod tests {
);
assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
}
fn image_url_part() -> MediaContentPart {
MediaContentPart::ImageUrl {
url: "https://example.com/image.png".to_string(),
detail: None,
uuid: None,
}
}
fn qwen3_vl_info_with_limits(limit_mm_per_prompt: MmLimitPerPrompt) -> MultimodalModelInfo {
test_info_with_limits(
"qwen3_vl",
qwen3_vl_config(),
qwen3_vl_tokenizer(),
limit_mm_per_prompt,
)
}
#[test]
fn validate_mm_limits_ignores_text_parts() {
let info = qwen3_vl_info();
let parts = vec![
MediaContentPart::Text {
text: "hello".to_string(),
},
MediaContentPart::Text {
text: "world".to_string(),
},
];
assert!(info.validate_mm_limits(&parts).is_ok());
}
#[test]
fn validate_mm_limits_leaves_unconfigured_modalities_unlimited() {
let info = qwen3_vl_info();
let parts: Vec<_> = std::iter::repeat_with(image_url_part).take(1_000).collect();
assert!(info.validate_mm_limits(&parts).is_ok());
}
#[test]
fn validate_mm_limits_enforces_configured_limit_at_the_boundary() {
let info = qwen3_vl_info_with_limits(HashMap::from([(
MmLimitModality::Image,
MmLimitSpec::Count(1),
)]));
assert!(info.validate_mm_limits(&[image_url_part()]).is_ok());
let error = info.validate_mm_limits(&[image_url_part(), image_url_part()]).unwrap_err();
assert_eq!(
error.to_report_string(),
"At most 1 image(s) may be provided in one prompt."
);
// Confirms the HTTP-mapping bug found during implementation stays fixed:
// this must map to 400, not 500.
assert!(error.is_request_validation_error());
}
#[test]
fn validate_mm_limits_counts_image_embeds_against_the_image_limit() {
let info = qwen3_vl_info_with_limits(HashMap::from([(
MmLimitModality::Image,
MmLimitSpec::Count(1),
)]));
let image_embeds_part = MediaContentPart::ImageEmbeds {
payload: serde_json::Value::String("AAAA".to_string()),
uuid: None,
};
let error = info.validate_mm_limits(&[image_url_part(), image_embeds_part]).unwrap_err();
assert_eq!(
error.to_report_string(),
"At most 1 image(s) may be provided in one prompt."
);
}
/// An options object without a `count` carries only profiling keys, which
/// say nothing about how many items are allowed.
#[test]
fn validate_mm_limits_treats_a_count_less_options_object_as_unlimited() {
let info = qwen3_vl_info_with_limits(HashMap::from([(
MmLimitModality::Image,
MmLimitSpec::Options {
count: None,
extra: BTreeMap::from([("width".to_string(), serde_json::json!(512))]),
},
)]));
assert!(info.validate_mm_limits(&[image_url_part(), image_url_part()]).is_ok());
}
fn parse_limits(json: &str) -> MmLimitPerPrompt {
serde_json::from_str(json).expect("limit map should parse")
}
#[test]
fn limit_map_parses_both_shapes_python_accepts() {
let limits = parse_limits(r#"{"image": 16, "video": {"count": 1, "num_frames": 32}}"#);
assert_eq!(limits[&MmLimitModality::Image].count(), Some(16));
assert_eq!(limits[&MmLimitModality::Video].count(), Some(1));
assert_eq!(limits.get(&MmLimitModality::Audio), None);
}
#[test]
fn limit_map_rejects_keys_python_does_not_accept() {
assert!(serde_json::from_str::<MmLimitPerPrompt>(r#"{"image_embeds": 1}"#).is_err());
}
/// Managed mode forwards this map back to Python as JSON, where
/// `BaseDummyOptions.count` is a non-optional `int` under `extra="forbid"`.
/// Emitting `"count": null` would make the engine subprocess fail to start.
#[test]
fn limit_map_round_trips_without_emitting_a_null_count() {
let source = r#"{"video":{"num_frames":32}}"#;
let encoded = serde_json::to_string(&parse_limits(source)).expect("map should serialize");
assert_eq!(encoded, source);
}
}
+2
View File
@@ -107,6 +107,7 @@ mod tests {
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
HashMap::new(),
)
.unwrap()
.expect("Inkling multimodal support")
@@ -126,6 +127,7 @@ mod tests {
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
HashMap::new(),
)
.unwrap()
.expect("Qwen3-ASR multimodal support")
+1
View File
@@ -207,6 +207,7 @@ mod tests {
Some("qwen3_vl".to_string()),
files,
Arc::new(qwen3_vl_tokenizer()),
std::collections::HashMap::new(),
)
};