[Rust Frontend] Add --limit-mm-per-prompt support (#49604)

Signed-off-by: Maria Guevara <kawaiiplush14@gmail.com>
This commit is contained in:
Maria Guevara
2026-07-29 17:21:41 +08:00
committed by GitHub
parent 5b14019576
commit c44e191b01
14 changed files with 413 additions and 16 deletions
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::time::Duration;
use anyhow::{Context, Result, bail};
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
language_model_only: false,
chat_template: None,
default_chat_template_kwargs: None,
limit_mm_per_prompt: HashMap::new(),
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_logprobs: None,
api_server_options: ApiServerOptions::default(),
+4
View File
@@ -10,6 +10,7 @@ use axum::http::{HeaderName, HeaderValue, Method};
use educe::Educe;
use serde::Serialize;
use serde_json::Value;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
@@ -184,6 +185,9 @@ pub struct Config {
pub chat_template: Option<String>,
/// Server-default keyword arguments merged into every chat-template render.
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// Maximum number of input items allowed per prompt for each modality.
/// Unspecified modalities are unlimited.
pub limit_mm_per_prompt: MmLimitPerPrompt,
/// How to serialize `message.content` for chat-template rendering.
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Optional maximum number of top log probabilities accepted by the
+1
View File
@@ -101,6 +101,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.default_chat_template_kwargs
.clone()
.unwrap_or_default(),
limit_mm_per_prompt: config.limit_mm_per_prompt.clone(),
},
)
.await
+75
View File
@@ -577,6 +577,12 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
}
fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
qwen_multimodal_model_info_with_limits(std::collections::HashMap::new())
}
fn qwen_multimodal_model_info_with_limits(
limit_mm_per_prompt: vllm_chat::multimodal::MmLimitPerPrompt,
) -> vllm_chat::multimodal::MultimodalModelInfo {
let config_path = std::env::temp_dir().join(format!(
"vllm-server-qwen-config-{}.json",
uuid::Uuid::new_v4()
@@ -594,6 +600,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
..Default::default()
},
Arc::new(fake_chat_tokenizer()),
limit_mm_per_prompt,
)
.expect("load multimodal info")
.expect("qwen multimodal info is registered");
@@ -2307,6 +2314,74 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() {
assert_eq!(json["choices"][0]["message"]["content"], "hi");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() {
// The request is rejected by `--limit-mm-per-prompt` validation before
// ever reaching the engine, so the mock engine task is never awaited.
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-openai-mm-limit",
default_stream_output_specs(),
Arc::new(FakeChatBackend::with_multimodal_model_info(
qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([(
vllm_chat::multimodal::MmLimitModality::Image,
vllm_chat::multimodal::MmLimitSpec::Count(1),
)])),
)),
)
.await;
let app = build_router(Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)));
let response = app
.clone()
.call(
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"stream": false,
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": "describe "},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
}
},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
}
}
]
}]
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["error"]["type"], "invalid_request_error");
assert_eq!(
json["error"]["message"],
"At most 1 image(s) may be provided in one prompt."
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {