[Core] Dynamic image size support for VLMs (#5276)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: ywang96 <ywang@roblox.com> Co-authored-by: xwjiang2010 <87673679+xwjiang2010@users.noreply.github.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
This commit is contained in:
co-authored by
Xiaowei Jiang
ywang96
xwjiang2010
Roger Wang
parent
482045ee77
commit
9831aec49f
@@ -1,6 +1,7 @@
|
||||
import codecs
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cached_property
|
||||
from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, Iterable,
|
||||
List, Optional)
|
||||
from typing import Sequence as GenericSequence
|
||||
@@ -10,7 +11,7 @@ from fastapi import Request
|
||||
from openai.types.chat import (ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam)
|
||||
|
||||
from vllm.config import ModelConfig, VisionLanguageConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionContentPartParam, ChatCompletionLogProb,
|
||||
@@ -27,8 +28,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.guided_decoding import (
|
||||
get_guided_decoding_logits_processor)
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.multimodal.utils import (async_get_and_parse_image,
|
||||
get_full_image_text_prompt)
|
||||
from vllm.multimodal.utils import async_get_and_parse_image
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sequence import Logprob
|
||||
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
|
||||
@@ -97,6 +97,36 @@ class OpenAIServingChat(OpenAIServing):
|
||||
logger.warning(
|
||||
"No chat template provided. Chat API will not work.")
|
||||
|
||||
@cached_property
|
||||
def image_token_str(self) -> Optional[str]:
|
||||
# TODO: Let user specify how to insert image tokens into prompt
|
||||
# (similar to chat template)
|
||||
model_type = self.model_config.hf_config.model_type
|
||||
if model_type == "phi3_v":
|
||||
# Workaround since this token is not defined in the tokenizer
|
||||
return "<|image_1|>"
|
||||
if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv",
|
||||
"paligemma"):
|
||||
# These models do not use image tokens in the prompt
|
||||
return None
|
||||
|
||||
# The default behaviour assumes that the image token is
|
||||
# available to the tokenizer.
|
||||
# (Suitable for LLaVA, Idefics2, DeepSeek-VL)
|
||||
vlm_config = self.model_config.multimodal_config
|
||||
if vlm_config is None:
|
||||
raise ValueError(
|
||||
"'image_url' input is not supported as the loaded "
|
||||
"model is not multimodal.")
|
||||
|
||||
image_token_id = vlm_config.image_token_id
|
||||
if vlm_config.image_token_id is None:
|
||||
raise ValueError(
|
||||
"'image_url' input is not supported as the loaded "
|
||||
"model does not specify an image token.")
|
||||
|
||||
return self.tokenizer.decode(image_token_id)
|
||||
|
||||
def _parse_chat_message_content_parts(
|
||||
self,
|
||||
role: str,
|
||||
@@ -105,21 +135,26 @@ class OpenAIServingChat(OpenAIServing):
|
||||
texts: List[str] = []
|
||||
mm_futures: List[Awaitable[MultiModalDataDict]] = []
|
||||
|
||||
vlm_config: Optional[VisionLanguageConfig] = getattr(
|
||||
self.engine.engine, "vision_language_config", None)
|
||||
model_config = getattr(self.engine.engine, "model_config", None)
|
||||
|
||||
for part in parts:
|
||||
part_type = part["type"]
|
||||
if part_type == "text":
|
||||
text = cast(ChatCompletionContentPartTextParam, part)["text"]
|
||||
texts.append(text)
|
||||
elif part_type == "image_url":
|
||||
if vlm_config is None:
|
||||
raise ValueError(
|
||||
"'image_url' input is not supported as the loaded "
|
||||
"model is not multimodal.")
|
||||
assert self.tokenizer is not None
|
||||
if len(mm_futures) > 0:
|
||||
raise NotImplementedError(
|
||||
"Multiple 'image_url' input is currently not supported."
|
||||
)
|
||||
|
||||
image_token_str = self.image_token_str
|
||||
if image_token_str is not None:
|
||||
if any(image_token_str in text for text in texts):
|
||||
logger.warning(
|
||||
"Detected image token string in the text prompt. "
|
||||
"Skipping prompt formatting.")
|
||||
else:
|
||||
texts.append(image_token_str)
|
||||
|
||||
image_url = cast(ChatCompletionContentPartImageParam,
|
||||
part)["image_url"]
|
||||
|
||||
@@ -128,43 +163,13 @@ class OpenAIServingChat(OpenAIServing):
|
||||
"'image_url.detail' is currently not supported and "
|
||||
"will be ignored.")
|
||||
|
||||
mm_future = async_get_and_parse_image(image_url["url"])
|
||||
mm_futures.append(mm_future)
|
||||
|
||||
image_future = async_get_and_parse_image(image_url["url"])
|
||||
mm_futures.append(image_future)
|
||||
else:
|
||||
raise NotImplementedError(f"Unknown part type: {part_type}")
|
||||
|
||||
text_prompt = "\n".join(texts)
|
||||
|
||||
if vlm_config is not None and len(mm_futures):
|
||||
|
||||
assert len(
|
||||
mm_futures
|
||||
) == 1, "Multiple 'image_url' input is currently not supported."
|
||||
(image_token_prompt,
|
||||
image_token_str) = vlm_config.get_image_token_text(self.tokenizer)
|
||||
|
||||
# NOTE: If image token string (e.g, <image>) is already present
|
||||
# in the text prompt, we assume it follows the same format required
|
||||
# by the engine.
|
||||
if image_token_str in text_prompt:
|
||||
logger.warning(
|
||||
"Detected image token string in the text prompt. "
|
||||
"Skipping prompt formatting.")
|
||||
messages = [
|
||||
ConversationMessage(role=role, content=text_prompt)
|
||||
]
|
||||
|
||||
else:
|
||||
full_prompt = get_full_image_text_prompt(
|
||||
image_prompt=image_token_prompt,
|
||||
text_prompt=text_prompt,
|
||||
config=model_config)
|
||||
messages = [
|
||||
ConversationMessage(role=role, content=full_prompt)
|
||||
]
|
||||
else:
|
||||
messages = [ConversationMessage(role=role, content=text_prompt)]
|
||||
messages = [ConversationMessage(role=role, content=text_prompt)]
|
||||
|
||||
return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
|
||||
|
||||
@@ -267,7 +272,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
"prompt": prompt_text,
|
||||
"prompt_token_ids": prompt_ids,
|
||||
}
|
||||
if mm_data is not None:
|
||||
if mm_data:
|
||||
inputs["multi_modal_data"] = mm_data
|
||||
|
||||
is_tracing_enabled = await self.engine.is_tracing_enabled()
|
||||
|
||||
@@ -36,6 +36,7 @@ class OpenAIServing:
|
||||
super().__init__()
|
||||
|
||||
self.engine = engine
|
||||
self.model_config = model_config
|
||||
self.max_model_len = model_config.max_model_len
|
||||
|
||||
# A separate tokenizer to map token IDs to strings.
|
||||
|
||||
Reference in New Issue
Block a user