[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-26 16:44:04 +08:00
committed by GitHub
parent 52069012fe
commit 502c41a8f6
12 changed files with 215 additions and 145 deletions
+11
View File
@@ -1210,6 +1210,17 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+11
View File
@@ -371,6 +371,17 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+11
View File
@@ -215,6 +215,17 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
grid_thws=MultiModalFieldConfig.batched("vision_chunk"),
)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path because vision chunk
# is not considered
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+23 -1
View File
@@ -12,7 +12,7 @@ import torch.nn.functional as F
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from transformers import PixtralVisionConfig
from transformers import BatchFeature, PixtralVisionConfig
from transformers.models.pixtral.image_processing_pixtral import (
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
)
@@ -62,6 +62,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.transformers_utils.processors.pixtral import MistralCommonPixtralProcessor
from vllm.utils.collection_utils import is_list_of
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (
@@ -213,6 +214,27 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]:
return dict(images=MultiModalFieldConfig.batched("image"))
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
# Avoid padding issue
tok_kwargs={**tok_kwargs, "return_tensors": None},
)
# Missing batch dimension
if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]]
return outputs
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+11
View File
@@ -929,6 +929,17 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_qwen2_5_vl.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+11 -3
View File
@@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.transformers_utils.processors.voxtral import MistralCommonVoxtralProcessor
from vllm.utils.collection_utils import is_list_of
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
from .utils import init_vllm_registered_model, maybe_prefix
@@ -208,7 +209,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
) -> None:
# mistral_common's tokenizer's does not follow HF's placeholder norms
# skip validation here
...
pass
def _call_hf_processor(
self,
@@ -224,13 +225,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
return super()._call_hf_processor(
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
# Avoid padding issue
tok_kwargs={**tok_kwargs, "return_tensors": None},
)
# Missing batch dimension
if is_list_of(outputs["input_ids"], int):
outputs["input_ids"] = [outputs["input_ids"]]
return outputs
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,