[Rust Frontend] Add --limit-mm-per-prompt support (#49604)
Signed-off-by: Maria Guevara <kawaiiplush14@gmail.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
limit_mm_per_prompt: HashMap::new(),
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
|
||||
@@ -10,6 +10,7 @@ use axum::http::{HeaderName, HeaderValue, Method};
|
||||
use educe::Educe;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
|
||||
@@ -184,6 +185,9 @@ pub struct Config {
|
||||
pub chat_template: Option<String>,
|
||||
/// Server-default keyword arguments merged into every chat-template render.
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// Maximum number of input items allowed per prompt for each modality.
|
||||
/// Unspecified modalities are unlimited.
|
||||
pub limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
/// How to serialize `message.content` for chat-template rendering.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Optional maximum number of top log probabilities accepted by the
|
||||
|
||||
@@ -101,6 +101,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.default_chat_template_kwargs
|
||||
.clone()
|
||||
.unwrap_or_default(),
|
||||
limit_mm_per_prompt: config.limit_mm_per_prompt.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -577,6 +577,12 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
|
||||
}
|
||||
|
||||
fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
qwen_multimodal_model_info_with_limits(std::collections::HashMap::new())
|
||||
}
|
||||
|
||||
fn qwen_multimodal_model_info_with_limits(
|
||||
limit_mm_per_prompt: vllm_chat::multimodal::MmLimitPerPrompt,
|
||||
) -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
let config_path = std::env::temp_dir().join(format!(
|
||||
"vllm-server-qwen-config-{}.json",
|
||||
uuid::Uuid::new_v4()
|
||||
@@ -594,6 +600,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(fake_chat_tokenizer()),
|
||||
limit_mm_per_prompt,
|
||||
)
|
||||
.expect("load multimodal info")
|
||||
.expect("qwen multimodal info is registered");
|
||||
@@ -2307,6 +2314,74 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() {
|
||||
assert_eq!(json["choices"][0]["message"]["content"], "hi");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() {
|
||||
// The request is rejected by `--limit-mm-per-prompt` validation before
|
||||
// ever reaching the engine, so the mock engine task is never awaited.
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-openai-mm-limit",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::with_multimodal_model_info(
|
||||
qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([(
|
||||
vllm_chat::multimodal::MmLimitModality::Image,
|
||||
vllm_chat::multimodal::MmLimitSpec::Count(1),
|
||||
)])),
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
let app = build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)));
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"stream": false,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe "},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
}
|
||||
}
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["error"]["type"], "invalid_request_error");
|
||||
assert_eq!(
|
||||
json["error"]["message"],
|
||||
"At most 1 image(s) may be provided in one prompt."
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
|
||||
|
||||
Reference in New Issue
Block a user