[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1210,6 +1210,17 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
|
||||
|
||||
|
||||
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_keye.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -371,6 +371,17 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
|
||||
|
||||
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_keye.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -215,6 +215,17 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
|
||||
grid_thws=MultiModalFieldConfig.batched("vision_chunk"),
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path because vision chunk
|
||||
# is not considered
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch.nn.functional as F
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
|
||||
from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
from transformers import PixtralVisionConfig
|
||||
from transformers import BatchFeature, PixtralVisionConfig
|
||||
from transformers.models.pixtral.image_processing_pixtral import (
|
||||
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
|
||||
)
|
||||
@@ -62,6 +62,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.transformers_utils.processors.pixtral import MistralCommonPixtralProcessor
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -213,6 +214,27 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(images=MultiModalFieldConfig.batched("image"))
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
# Avoid padding issue
|
||||
tok_kwargs={**tok_kwargs, "return_tensors": None},
|
||||
)
|
||||
|
||||
# Missing batch dimension
|
||||
if is_list_of(outputs["input_ids"], int):
|
||||
outputs["input_ids"] = [outputs["input_ids"]]
|
||||
|
||||
return outputs
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -929,6 +929,17 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_qwen2_5_vl.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.transformers_utils.processors.voxtral import MistralCommonVoxtralProcessor
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
|
||||
from .utils import init_vllm_registered_model, maybe_prefix
|
||||
@@ -208,7 +209,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
) -> None:
|
||||
# mistral_common's tokenizer's does not follow HF's placeholder norms
|
||||
# skip validation here
|
||||
...
|
||||
pass
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
@@ -224,13 +225,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
# MistralCommonVoxtralProcessor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
|
||||
return super()._call_hf_processor(
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
# Avoid padding issue
|
||||
tok_kwargs={**tok_kwargs, "return_tensors": None},
|
||||
)
|
||||
|
||||
# Missing batch dimension
|
||||
if is_list_of(outputs["input_ids"], int):
|
||||
outputs["input_ids"] = [outputs["input_ids"]]
|
||||
|
||||
return outputs
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
Reference in New Issue
Block a user