[Rust Frontend] Add --limit-mm-per-prompt support (#49604)
Signed-off-by: Maria Guevara <kawaiiplush14@gmail.com>
This commit is contained in:
@@ -57,6 +57,7 @@ impl HfChatBackend {
|
||||
processor_config: files.processor_config_path.as_deref(),
|
||||
},
|
||||
tokenizer.clone(),
|
||||
options.limit_mm_per_prompt.clone(),
|
||||
)?
|
||||
};
|
||||
let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());
|
||||
@@ -231,6 +232,7 @@ mod tests {
|
||||
chat_template_content_format: Default::default(),
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: HashMap::new(),
|
||||
limit_mm_per_prompt: HashMap::new(),
|
||||
},
|
||||
test_tokenizer(),
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ use serde_json::Value;
|
||||
use vllm_text::{DynTextBackend, TextBackend};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::multimodal::MultimodalModelInfo;
|
||||
use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
|
||||
use crate::output::DynChatOutputProcessor;
|
||||
use crate::renderer::DynChatRenderer;
|
||||
use crate::request::ChatRequest;
|
||||
@@ -74,6 +74,9 @@ pub struct LoadModelBackendsOptions {
|
||||
/// Optional server-default keyword arguments merged into every
|
||||
/// chat-template render before request-level `chat_template_kwargs`.
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
/// Maximum number of input items allowed per prompt for each modality.
|
||||
/// Unspecified modalities are unlimited.
|
||||
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
}
|
||||
|
||||
/// Shared backends loaded from a model id.
|
||||
|
||||
@@ -23,6 +23,8 @@ pub enum Error {
|
||||
UnsupportedMultimodalContent(&'static str),
|
||||
#[error("`{modality}` input is not supported by this model")]
|
||||
UnsupportedModality { modality: String },
|
||||
#[error("At most {limit} {modality}(s) may be provided in one prompt.")]
|
||||
MmLimitExceeded { modality: String, limit: usize },
|
||||
#[error("multimodal preprocessing error: {0}")]
|
||||
Multimodal(#[message] String),
|
||||
#[error("{kind} parsing is not available for model `{model_id}`")]
|
||||
@@ -87,7 +89,8 @@ impl Error {
|
||||
Self::Text(error) => error.is_request_validation_error(),
|
||||
Self::UnsupportedMultimodalRenderer
|
||||
| Self::UnsupportedMultimodalContent(_)
|
||||
| Self::UnsupportedModality { .. } => true,
|
||||
| Self::UnsupportedModality { .. }
|
||||
| Self::MmLimitExceeded { .. } => true,
|
||||
|
||||
_ => false,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
//! Raw media stays above `vllm-text`; this module lowers it into token IDs and
|
||||
//! opaque tensor payloads before the request is handed to text generation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
@@ -24,6 +24,7 @@ use llm_multimodal::{
|
||||
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
|
||||
VisionProcessorRegistry,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::warn;
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
@@ -52,6 +53,71 @@ pub struct MultimodalModelInfo {
|
||||
video: Option<ModalitySupport>,
|
||||
audio: Option<AudioModalitySupport>,
|
||||
media_connector: Arc<MediaConnector>,
|
||||
/// Maximum number of input items allowed per prompt for each modality.
|
||||
limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
}
|
||||
|
||||
/// Per-modality item-count limits configured by `--limit-mm-per-prompt`.
|
||||
///
|
||||
/// Modalities absent from the map are unlimited.
|
||||
pub type MmLimitPerPrompt = HashMap<MmLimitModality, MmLimitSpec>;
|
||||
|
||||
/// Modalities that `--limit-mm-per-prompt` can be keyed by.
|
||||
///
|
||||
/// Closed on purpose: these are exactly the keys Python accepts, per
|
||||
/// `MultiModalDummyOptionsBuiltins` in `vllm/config/multimodal.py`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MmLimitModality {
|
||||
Image,
|
||||
Audio,
|
||||
Video,
|
||||
}
|
||||
|
||||
impl MmLimitModality {
|
||||
/// The wire name, matching Python's modality strings.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Image => "image",
|
||||
Self::Audio => "audio",
|
||||
Self::Video => "video",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One modality's limit, in either of the two shapes Python accepts.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(
|
||||
untagged,
|
||||
expecting = "an item count, or an object with an optional `count` field"
|
||||
)]
|
||||
pub enum MmLimitSpec {
|
||||
/// Legacy form: `"image": 16`
|
||||
Count(usize),
|
||||
|
||||
/// Configurable form:
|
||||
/// `"video": {"count": 1, "num_frames": 32}`
|
||||
Options {
|
||||
/// Absent means unlimited, matching an absent modality.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
count: Option<usize>,
|
||||
|
||||
/// Preserve Python-owned options for forwarding. Never interpreted
|
||||
/// here: they size the engine's dummy-profiling encoder cache, which
|
||||
/// has no Rust counterpart.
|
||||
#[serde(flatten)]
|
||||
extra: BTreeMap<String, serde_json::Value>,
|
||||
},
|
||||
}
|
||||
|
||||
impl MmLimitSpec {
|
||||
/// The configured item count, or `None` when this modality is unlimited.
|
||||
pub fn count(&self) -> Option<usize> {
|
||||
match self {
|
||||
Self::Count(count) => Some(*count),
|
||||
Self::Options { count, .. } => *count,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Model metadata and tokenizer access shared by all multimodal specs.
|
||||
@@ -287,6 +353,7 @@ impl MultimodalModelInfo {
|
||||
model_type: Option<String>,
|
||||
files: MultimodalConfigFiles<'_>,
|
||||
tokenizer: DynTokenizer,
|
||||
limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
) -> Result<Option<Self>> {
|
||||
let config = match files.config {
|
||||
Some(path) => {
|
||||
@@ -319,7 +386,12 @@ impl MultimodalModelInfo {
|
||||
tokenizer: TokenizerResolver(tokenizer),
|
||||
};
|
||||
|
||||
Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
|
||||
Self::from_loaded(
|
||||
context,
|
||||
preprocessor_config,
|
||||
video_preprocessor_config,
|
||||
limit_mm_per_prompt,
|
||||
)
|
||||
}
|
||||
|
||||
/// Resolve multimodal support from an assembled context and parsed
|
||||
@@ -328,6 +400,7 @@ impl MultimodalModelInfo {
|
||||
context: MultimodalModelContext,
|
||||
preprocessor_config: PreProcessorConfig,
|
||||
video_preprocessor_config: PreProcessorConfig,
|
||||
limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
) -> Result<Option<Self>> {
|
||||
let (image, video) = Self::resolve_vision_lanes(
|
||||
&context,
|
||||
@@ -356,6 +429,7 @@ impl MultimodalModelInfo {
|
||||
video,
|
||||
audio,
|
||||
media_connector,
|
||||
limit_mm_per_prompt,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -567,7 +641,55 @@ fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
|
||||
Ok(format!("data:{mime_type};base64,{data}"))
|
||||
}
|
||||
|
||||
/// The modality a content part counts against, or `None` for plain text.
|
||||
///
|
||||
/// Embedding inputs share their base modality's budget rather than getting one
|
||||
/// of their own, matching Python's `modality.replace("_embeds", "")` in
|
||||
/// `vllm/entrypoints/chat_utils.py`.
|
||||
fn media_part_limit_modality(part: &MediaContentPart) -> Option<MmLimitModality> {
|
||||
match part {
|
||||
MediaContentPart::Text { .. } => None,
|
||||
MediaContentPart::ImageUrl { .. }
|
||||
| MediaContentPart::ImageData { .. }
|
||||
| MediaContentPart::ImageEmbeds { .. } => Some(MmLimitModality::Image),
|
||||
MediaContentPart::AudioUrl { .. } | MediaContentPart::AudioData { .. } => {
|
||||
Some(MmLimitModality::Audio)
|
||||
}
|
||||
MediaContentPart::VideoUrl { .. } | MediaContentPart::VideoData { .. } => {
|
||||
Some(MmLimitModality::Video)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MultimodalModelInfo {
|
||||
/// Reject requests exceeding `--limit-mm-per-prompt`'s configured
|
||||
/// per-modality item count, before any fetch/decode work is spent on them.
|
||||
///
|
||||
/// Modalities without a configured count are unlimited.
|
||||
fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> {
|
||||
let mut counts: HashMap<MmLimitModality, usize> = HashMap::new();
|
||||
for part in media_parts {
|
||||
if let Some(modality) = media_part_limit_modality(part) {
|
||||
*counts.entry(modality).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (modality, count) in counts {
|
||||
let Some(limit) = self.limit_mm_per_prompt.get(&modality).and_then(MmLimitSpec::count)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if count > limit {
|
||||
return Err(Error::MmLimitExceeded {
|
||||
modality: modality.as_str().to_string(),
|
||||
limit,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run media fetch, per-modality preprocessing, prompt expansion, and
|
||||
/// feature build.
|
||||
///
|
||||
@@ -584,8 +706,7 @@ impl MultimodalModelInfo {
|
||||
if media_parts_len == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
// TODO: enforce per-modality item-count limits, aligned with the
|
||||
// engine's `--limit-mm-per-prompt` semantics.
|
||||
self.validate_mm_limits(&media_parts)?;
|
||||
let fetched = self.fetch_media(media_parts).await?;
|
||||
|
||||
let mut prepared = Vec::new();
|
||||
@@ -753,10 +874,11 @@ mod tests {
|
||||
.with_regular_token("<|video_pad|>", QWEN3_VIDEO_PAD_ID)
|
||||
}
|
||||
|
||||
fn test_info(
|
||||
fn test_info_with_limits(
|
||||
model_type: &str,
|
||||
config: serde_json::Value,
|
||||
tokenizer: TestTokenizer,
|
||||
limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
) -> MultimodalModelInfo {
|
||||
let context = MultimodalModelContext {
|
||||
model_id: format!("{model_type}-test"),
|
||||
@@ -769,11 +891,20 @@ mod tests {
|
||||
context,
|
||||
PreProcessorConfig::default(),
|
||||
PreProcessorConfig::default(),
|
||||
limit_mm_per_prompt,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| panic!("{model_type} multimodal support should resolve"))
|
||||
}
|
||||
|
||||
fn test_info(
|
||||
model_type: &str,
|
||||
config: serde_json::Value,
|
||||
tokenizer: TestTokenizer,
|
||||
) -> MultimodalModelInfo {
|
||||
test_info_with_limits(model_type, config, tokenizer, HashMap::new())
|
||||
}
|
||||
|
||||
fn llama4_info() -> MultimodalModelInfo {
|
||||
let config = serde_json::json!({
|
||||
"model_type": "llama4",
|
||||
@@ -783,16 +914,19 @@ mod tests {
|
||||
test_info("llama4", config, llama4_tokenizer())
|
||||
}
|
||||
|
||||
pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
|
||||
let config = serde_json::json!({
|
||||
fn qwen3_vl_config() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"model_type": "qwen3_vl",
|
||||
"image_token_id": QWEN3_IMAGE_PAD_ID,
|
||||
"video_token_id": QWEN3_VIDEO_PAD_ID,
|
||||
"vision_start_token_id": 151652,
|
||||
"vision_end_token_id": 151653,
|
||||
"vision_config": {"patch_size": 16}
|
||||
});
|
||||
test_info("qwen3_vl", config, qwen3_vl_tokenizer())
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
|
||||
test_info("qwen3_vl", qwen3_vl_config(), qwen3_vl_tokenizer())
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -853,4 +987,123 @@ mod tests {
|
||||
);
|
||||
assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
|
||||
}
|
||||
|
||||
fn image_url_part() -> MediaContentPart {
|
||||
MediaContentPart::ImageUrl {
|
||||
url: "https://example.com/image.png".to_string(),
|
||||
detail: None,
|
||||
uuid: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn qwen3_vl_info_with_limits(limit_mm_per_prompt: MmLimitPerPrompt) -> MultimodalModelInfo {
|
||||
test_info_with_limits(
|
||||
"qwen3_vl",
|
||||
qwen3_vl_config(),
|
||||
qwen3_vl_tokenizer(),
|
||||
limit_mm_per_prompt,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mm_limits_ignores_text_parts() {
|
||||
let info = qwen3_vl_info();
|
||||
let parts = vec![
|
||||
MediaContentPart::Text {
|
||||
text: "hello".to_string(),
|
||||
},
|
||||
MediaContentPart::Text {
|
||||
text: "world".to_string(),
|
||||
},
|
||||
];
|
||||
assert!(info.validate_mm_limits(&parts).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mm_limits_leaves_unconfigured_modalities_unlimited() {
|
||||
let info = qwen3_vl_info();
|
||||
let parts: Vec<_> = std::iter::repeat_with(image_url_part).take(1_000).collect();
|
||||
|
||||
assert!(info.validate_mm_limits(&parts).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mm_limits_enforces_configured_limit_at_the_boundary() {
|
||||
let info = qwen3_vl_info_with_limits(HashMap::from([(
|
||||
MmLimitModality::Image,
|
||||
MmLimitSpec::Count(1),
|
||||
)]));
|
||||
assert!(info.validate_mm_limits(&[image_url_part()]).is_ok());
|
||||
|
||||
let error = info.validate_mm_limits(&[image_url_part(), image_url_part()]).unwrap_err();
|
||||
assert_eq!(
|
||||
error.to_report_string(),
|
||||
"At most 1 image(s) may be provided in one prompt."
|
||||
);
|
||||
// Confirms the HTTP-mapping bug found during implementation stays fixed:
|
||||
// this must map to 400, not 500.
|
||||
assert!(error.is_request_validation_error());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mm_limits_counts_image_embeds_against_the_image_limit() {
|
||||
let info = qwen3_vl_info_with_limits(HashMap::from([(
|
||||
MmLimitModality::Image,
|
||||
MmLimitSpec::Count(1),
|
||||
)]));
|
||||
let image_embeds_part = MediaContentPart::ImageEmbeds {
|
||||
payload: serde_json::Value::String("AAAA".to_string()),
|
||||
uuid: None,
|
||||
};
|
||||
|
||||
let error = info.validate_mm_limits(&[image_url_part(), image_embeds_part]).unwrap_err();
|
||||
assert_eq!(
|
||||
error.to_report_string(),
|
||||
"At most 1 image(s) may be provided in one prompt."
|
||||
);
|
||||
}
|
||||
|
||||
/// An options object without a `count` carries only profiling keys, which
|
||||
/// say nothing about how many items are allowed.
|
||||
#[test]
|
||||
fn validate_mm_limits_treats_a_count_less_options_object_as_unlimited() {
|
||||
let info = qwen3_vl_info_with_limits(HashMap::from([(
|
||||
MmLimitModality::Image,
|
||||
MmLimitSpec::Options {
|
||||
count: None,
|
||||
extra: BTreeMap::from([("width".to_string(), serde_json::json!(512))]),
|
||||
},
|
||||
)]));
|
||||
|
||||
assert!(info.validate_mm_limits(&[image_url_part(), image_url_part()]).is_ok());
|
||||
}
|
||||
|
||||
fn parse_limits(json: &str) -> MmLimitPerPrompt {
|
||||
serde_json::from_str(json).expect("limit map should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_map_parses_both_shapes_python_accepts() {
|
||||
let limits = parse_limits(r#"{"image": 16, "video": {"count": 1, "num_frames": 32}}"#);
|
||||
|
||||
assert_eq!(limits[&MmLimitModality::Image].count(), Some(16));
|
||||
assert_eq!(limits[&MmLimitModality::Video].count(), Some(1));
|
||||
assert_eq!(limits.get(&MmLimitModality::Audio), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_map_rejects_keys_python_does_not_accept() {
|
||||
assert!(serde_json::from_str::<MmLimitPerPrompt>(r#"{"image_embeds": 1}"#).is_err());
|
||||
}
|
||||
|
||||
/// Managed mode forwards this map back to Python as JSON, where
|
||||
/// `BaseDummyOptions.count` is a non-optional `int` under `extra="forbid"`.
|
||||
/// Emitting `"count": null` would make the engine subprocess fail to start.
|
||||
#[test]
|
||||
fn limit_map_round_trips_without_emitting_a_null_count() {
|
||||
let source = r#"{"video":{"num_frames":32}}"#;
|
||||
let encoded = serde_json::to_string(&parse_limits(source)).expect("map should serialize");
|
||||
|
||||
assert_eq!(encoded, source);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,6 +107,7 @@ mod tests {
|
||||
context,
|
||||
PreProcessorConfig::default(),
|
||||
PreProcessorConfig::default(),
|
||||
HashMap::new(),
|
||||
)
|
||||
.unwrap()
|
||||
.expect("Inkling multimodal support")
|
||||
@@ -126,6 +127,7 @@ mod tests {
|
||||
context,
|
||||
PreProcessorConfig::default(),
|
||||
PreProcessorConfig::default(),
|
||||
HashMap::new(),
|
||||
)
|
||||
.unwrap()
|
||||
.expect("Qwen3-ASR multimodal support")
|
||||
|
||||
@@ -207,6 +207,7 @@ mod tests {
|
||||
Some("qwen3_vl".to_string()),
|
||||
files,
|
||||
Arc::new(qwen3_vl_tokenizer()),
|
||||
std::collections::HashMap::new(),
|
||||
)
|
||||
};
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::ReasoningParserFactory;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
@@ -186,6 +187,17 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// The maximum number of input items allowed per prompt for each
|
||||
/// modality, as a JSON object (e.g. `{"image": 16, "video": 2}`).
|
||||
///
|
||||
/// Also accepts the engine's configurable form
|
||||
/// (e.g. `{"video": {"count": 1, "num_frames": 32}}`); the extra
|
||||
/// profiling options are forwarded to the engine untouched.
|
||||
/// Unspecified modalities are unlimited.
|
||||
#[arg(long, value_parser = parse_json::<MmLimitPerPrompt>, value_name = "JSON", default_value = "{}")]
|
||||
#[serde(default)]
|
||||
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
|
||||
/// The format to render message content within a chat template.
|
||||
///
|
||||
/// * "auto" detects the format from the template
|
||||
@@ -348,6 +360,18 @@ impl SharedRuntimeArgs {
|
||||
.expect("profiler config serialization should not fail")
|
||||
}
|
||||
|
||||
/// Return the per-modality limits as JSON for managed Python engine
|
||||
/// forwarding, or `None` when nothing is configured.
|
||||
///
|
||||
/// Round-tripping the parsed map rather than the raw argument keeps the
|
||||
/// engine's own profiling options (`num_frames`, `width`, ...) intact.
|
||||
pub fn limit_mm_per_prompt_json(&self) -> Option<String> {
|
||||
(!self.limit_mm_per_prompt.is_empty()).then(|| {
|
||||
serde_json::to_string(&self.limit_mm_per_prompt)
|
||||
.expect("limit-mm-per-prompt serialization should not fail")
|
||||
})
|
||||
}
|
||||
|
||||
/// Apply fallback logic for API key configuration from env variables.
|
||||
fn apply_env_api_key_fallback(&mut self) {
|
||||
if self.api_key.is_empty()
|
||||
@@ -399,6 +423,7 @@ impl SharedRuntimeArgs {
|
||||
language_model_only: self.language_model_only,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
limit_mm_per_prompt: self.limit_mm_per_prompt,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
@@ -451,6 +476,7 @@ impl SharedRuntimeArgs {
|
||||
language_model_only: self.language_model_only,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
limit_mm_per_prompt: self.limit_mm_per_prompt,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
@@ -666,6 +692,7 @@ impl ServeArgs {
|
||||
self.runtime.disable_log_stats,
|
||||
self.runtime.shutdown_timeout,
|
||||
handshake_port,
|
||||
self.runtime.limit_mm_per_prompt_json(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,6 +64,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: {},
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -762,6 +763,7 @@ fn frontend_args_accept_json() {
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: {},
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -1117,6 +1119,24 @@ fn frontend_args_json_rejects_malformed_json() {
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_modality_in_limit_mm_per_prompt() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--limit-mm-per-prompt",
|
||||
r#"{"unsupported_modality": 1}"#,
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '{"unsupported_modality": 1}' for '--limit-mm-per-prompt <JSON>': invalid JSON object: unknown variant `unsupported_modality`, expected one of `image`, `audio`, `video` at line 1 column 23
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_flags_before_model() {
|
||||
let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
|
||||
@@ -1331,6 +1351,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: {},
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -1474,6 +1495,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: {},
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
@@ -1558,6 +1580,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: {},
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
@@ -1660,6 +1683,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: {},
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
|
||||
@@ -299,11 +299,6 @@ pub struct EngineUnsupportedArgs {
|
||||
)]
|
||||
pub kv_sharing_fast_prefill: Option<Unsupported>,
|
||||
|
||||
/// The maximum number of input items and options allowed per
|
||||
/// prompt for each modality.
|
||||
#[arg(long)]
|
||||
pub limit_mm_per_prompt: Option<Unsupported>,
|
||||
|
||||
/// Additional args passed to process media inputs, keyed by modalities.
|
||||
#[arg(long)]
|
||||
pub media_io_kwargs: Option<Unsupported>,
|
||||
|
||||
@@ -91,6 +91,7 @@ impl ManagedEngineArgs {
|
||||
disable_log_stats: bool,
|
||||
shutdown_timeout: u64,
|
||||
handshake_port: u16,
|
||||
limit_mm_per_prompt: Option<String>,
|
||||
) -> ManagedEngineConfig {
|
||||
let mut python_args = self.python_args;
|
||||
// Manually forward some args to the Python engine.
|
||||
@@ -126,6 +127,10 @@ impl ManagedEngineArgs {
|
||||
python_args.push("--data-parallel-size-local".to_string());
|
||||
python_args.push(data_parallel_size_local.to_string());
|
||||
}
|
||||
if let Some(limit_mm_per_prompt) = limit_mm_per_prompt {
|
||||
python_args.push("--limit-mm-per-prompt".to_string());
|
||||
python_args.push(limit_mm_per_prompt);
|
||||
}
|
||||
|
||||
ManagedEngineConfig {
|
||||
python: self.python,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: HashMap::new(),
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
|
||||
@@ -10,6 +10,7 @@ use axum::http::{HeaderName, HeaderValue, Method};
|
||||
use educe::Educe;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
|
||||
@@ -184,6 +185,9 @@ pub struct Config {
|
||||
pub chat_template: Option<String>,
|
||||
/// Server-default keyword arguments merged into every chat-template render.
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// Maximum number of input items allowed per prompt for each modality.
|
||||
/// Unspecified modalities are unlimited.
|
||||
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
/// How to serialize `message.content` for chat-template rendering.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Optional maximum number of top log probabilities accepted by the
|
||||
|
||||
@@ -101,6 +101,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.default_chat_template_kwargs
|
||||
.clone()
|
||||
.unwrap_or_default(),
|
||||
limit_mm_per_prompt: config.limit_mm_per_prompt.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -577,6 +577,12 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
|
||||
}
|
||||
|
||||
fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
qwen_multimodal_model_info_with_limits(std::collections::HashMap::new())
|
||||
}
|
||||
|
||||
fn qwen_multimodal_model_info_with_limits(
|
||||
limit_mm_per_prompt: vllm_chat::multimodal::MmLimitPerPrompt,
|
||||
) -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
let config_path = std::env::temp_dir().join(format!(
|
||||
"vllm-server-qwen-config-{}.json",
|
||||
uuid::Uuid::new_v4()
|
||||
@@ -594,6 +600,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(fake_chat_tokenizer()),
|
||||
limit_mm_per_prompt,
|
||||
)
|
||||
.expect("load multimodal info")
|
||||
.expect("qwen multimodal info is registered");
|
||||
@@ -2307,6 +2314,74 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() {
|
||||
assert_eq!(json["choices"][0]["message"]["content"], "hi");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() {
|
||||
// The request is rejected by `--limit-mm-per-prompt` validation before
|
||||
// ever reaching the engine, so the mock engine task is never awaited.
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-openai-mm-limit",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::with_multimodal_model_info(
|
||||
qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([(
|
||||
vllm_chat::multimodal::MmLimitModality::Image,
|
||||
vllm_chat::multimodal::MmLimitSpec::Count(1),
|
||||
)])),
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
let app = build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)));
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"stream": false,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe "},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
}
|
||||
}
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["error"]["type"], "invalid_request_error");
|
||||
assert_eq!(
|
||||
json["error"]["message"],
|
||||
"At most 1 image(s) may be provided in one prompt."
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
|
||||
|
||||
Reference in New Issue
Block a user