[Core] Move multimodal placeholder from chat utils to model definition (#20355)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -6,7 +6,7 @@ import json
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Awaitable, Iterable
|
||||
from functools import cache, lru_cache, partial
|
||||
from functools import cached_property, lru_cache, partial
|
||||
from pathlib import Path
|
||||
from typing import (Any, Callable, Generic, Literal, Optional, TypeVar, Union,
|
||||
cast)
|
||||
@@ -37,6 +37,8 @@ from typing_extensions import Required, TypeAlias, TypedDict
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
from vllm.model_executor.models import SupportsMultiModal
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
|
||||
from vllm.multimodal.utils import MediaConnector
|
||||
# yapf: disable
|
||||
@@ -492,6 +494,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._model_config
|
||||
|
||||
@cached_property
|
||||
def model_cls(self):
|
||||
return get_model_cls(self.model_config)
|
||||
|
||||
@property
|
||||
def allowed_local_media_path(self):
|
||||
return self._model_config.allowed_local_media_path
|
||||
@@ -500,89 +506,6 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
def mm_registry(self):
|
||||
return MULTIMODAL_REGISTRY
|
||||
|
||||
@staticmethod
|
||||
@cache
|
||||
def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
|
||||
return tokenizer.decode(token_index)
|
||||
|
||||
def _placeholder_str(self, modality: ModalityStr,
|
||||
current_count: int) -> Optional[str]:
|
||||
if modality in self._model_config.mm_placeholder_str_override:
|
||||
return self._model_config.mm_placeholder_str_override[modality]
|
||||
|
||||
# TODO: Let user specify how to insert image tokens into prompt
|
||||
# (similar to chat template)
|
||||
hf_config = self._model_config.hf_config
|
||||
model_type = hf_config.model_type
|
||||
|
||||
if modality in ("image", "image_embeds"):
|
||||
if model_type == "chatglm":
|
||||
return "<|begin_of_image|><|endoftext|><|end_of_image|>"
|
||||
if model_type == "glm4v":
|
||||
return "<|begin_of_image|><|image|><|end_of_image|>"
|
||||
if model_type in ("phi3_v", "phi4mm"):
|
||||
return f"<|image_{current_count}|>"
|
||||
if model_type in ("minicpmo", "minicpmv"):
|
||||
return "(<image>./</image>)"
|
||||
if model_type in ("blip-2", "florence2", "fuyu", "paligemma",
|
||||
"pixtral", "mistral3"):
|
||||
# These models do not use image tokens in the prompt
|
||||
return None
|
||||
if model_type == "qwen":
|
||||
return f"Picture {current_count}: <img></img>"
|
||||
if model_type.startswith("llava"):
|
||||
return self._cached_token_str(self._tokenizer,
|
||||
hf_config.image_token_index)
|
||||
|
||||
if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",
|
||||
"internvl_chat", "ovis", "skywork_chat",
|
||||
"NVLM_D", "h2ovl_chat", "idefics3", "smolvlm"):
|
||||
return "<image>"
|
||||
if model_type in ("mllama", "llama4"):
|
||||
return "<|image|>"
|
||||
if model_type in ("qwen2_vl", "qwen2_5_vl", "keye", "Keye"):
|
||||
return "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
if model_type == "qwen2_5_omni":
|
||||
return "<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
if model_type == "molmo":
|
||||
return ""
|
||||
if model_type == "aria":
|
||||
return "<|fim_prefix|><|img|><|fim_suffix|>"
|
||||
if model_type == "gemma3":
|
||||
return "<start_of_image>"
|
||||
if model_type == "kimi_vl":
|
||||
return "<|media_start|>image<|media_content|><|media_pad|><|media_end|>" # noqa: E501
|
||||
|
||||
raise TypeError(f"Unknown {modality} model type: {model_type}")
|
||||
elif modality == "audio":
|
||||
if model_type in ("ultravox", "granite_speech"):
|
||||
return "<|audio|>"
|
||||
if model_type == "phi4mm":
|
||||
return f"<|audio_{current_count}|>"
|
||||
if model_type in ("qwen2_audio", "qwen2_5_omni"):
|
||||
return (f"Audio {current_count}: "
|
||||
f"<|audio_bos|><|AUDIO|><|audio_eos|>")
|
||||
if model_type == "minicpmo":
|
||||
return "(<audio>./</audio>)"
|
||||
raise TypeError(f"Unknown model type: {model_type}")
|
||||
elif modality == "video":
|
||||
if model_type == "internvl_chat":
|
||||
return "<video>"
|
||||
if model_type == "glm4v":
|
||||
return "<|begin_of_video|><|video|><|end_of_video|>"
|
||||
if model_type in ("qwen2_vl", "qwen2_5_vl", "keye", "Keye"):
|
||||
return "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
if model_type == "qwen2_5_omni":
|
||||
return "<|vision_start|><|VIDEO|><|vision_end|>"
|
||||
if model_type in ("minicpmo", "minicpmv"):
|
||||
return "(<video>./</video>)"
|
||||
if model_type.startswith("llava"):
|
||||
return self._cached_token_str(self._tokenizer,
|
||||
hf_config.video_token_index)
|
||||
raise TypeError(f"Unknown {modality} model type: {model_type}")
|
||||
else:
|
||||
raise TypeError(f"Unknown modality: {modality}")
|
||||
|
||||
def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
|
||||
"""
|
||||
Add a multi-modal item to the current prompt and returns the
|
||||
@@ -590,6 +513,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
"""
|
||||
mm_registry = self.mm_registry
|
||||
model_config = self.model_config
|
||||
model_cls = cast(SupportsMultiModal, self.model_cls)
|
||||
|
||||
input_modality = modality.replace("_embeds", "")
|
||||
|
||||
@@ -614,7 +538,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
|
||||
self._items_by_modality[modality].append(item)
|
||||
|
||||
return self._placeholder_str(modality, current_count)
|
||||
return model_cls.get_placeholder_str(modality, current_count)
|
||||
|
||||
@abstractmethod
|
||||
def create_parser(self) -> "BaseMultiModalContentParser":
|
||||
|
||||
@@ -5,6 +5,7 @@ import io
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from functools import cached_property
|
||||
from math import ceil
|
||||
from typing import Callable, Literal, Optional, TypeVar, Union, cast
|
||||
|
||||
@@ -24,7 +25,8 @@ from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
|
||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.utils import get_model_architecture
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
from vllm.model_executor.models import SupportsTranscription
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.transformers_utils.processor import cached_get_processor
|
||||
from vllm.utils import PlaceholderModule
|
||||
@@ -76,24 +78,29 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
self.model_sr = processor.feature_extractor.sampling_rate
|
||||
self.hop_length = processor.feature_extractor.hop_length
|
||||
self.task_type = task_type
|
||||
self.model_cls, _ = get_model_architecture(model_config)
|
||||
|
||||
if self.default_sampling_params:
|
||||
logger.info(
|
||||
"Overwriting default completion sampling param with: %s",
|
||||
self.default_sampling_params)
|
||||
|
||||
@cached_property
|
||||
def model_cls(self):
|
||||
return get_model_cls(self.model_config)
|
||||
|
||||
async def _preprocess_speech_to_text(
|
||||
self,
|
||||
request: SpeechToTextRequest,
|
||||
audio_data: bytes,
|
||||
) -> tuple[list[PromptType], float]:
|
||||
model_cls = cast(SupportsTranscription, self.model_cls)
|
||||
|
||||
# Validate request
|
||||
# TODO language should be optional and can be guessed.
|
||||
# For now we default to en. See
|
||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
|
||||
lang = request.language or "en"
|
||||
self.model_cls.validate_language(lang) # type: ignore[attr-defined]
|
||||
model_cls.validate_language(lang)
|
||||
|
||||
if len(audio_data) / 1024**2 > MAX_AUDIO_CLIP_FILESIZE_MB:
|
||||
raise ValueError("Maximum file size exceeded.")
|
||||
@@ -117,9 +124,8 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
},
|
||||
},
|
||||
"decoder_prompt":
|
||||
self.model_cls.
|
||||
get_decoder_prompt( # type: ignore[attr-defined]
|
||||
lang, self.task_type, request.prompt)
|
||||
model_cls.get_decoder_prompt(lang, self.task_type,
|
||||
request.prompt)
|
||||
}
|
||||
prompts.append(cast(PromptType, prompt))
|
||||
return prompts, duration
|
||||
|
||||
Reference in New Issue
Block a user