[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
+18 -39
View File
@@ -16,7 +16,7 @@ from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from mistral_common.protocol.transcription.request import TranscriptionRequest
from mistral_common.tokens.tokenizers.audio import Audio
from transformers import BatchFeature, WhisperConfig
from transformers import WhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
@@ -36,7 +36,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
NestedTensors,
)
from vllm.multimodal.parse import (
@@ -48,7 +47,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
MultiModalPromptUpdates,
HFMultiModalInputs,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
@@ -61,7 +61,6 @@ from vllm.transformers_utils.processors.voxtral import (
MistralCommonFeatureExtractor,
MistralCommonVoxtralProcessor,
)
from vllm.utils.collection_utils import is_list_of
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
from .utils import init_vllm_registered_model, maybe_prefix
@@ -209,21 +208,19 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
mm_placeholders = self._find_mm_placeholders(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
return prompt_ids, mm_placeholders
return mm_res.prompt_ids, mm_placeholders
def _get_mm_fields_config(
self,
@@ -241,38 +238,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# skip validation here
pass
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
# Avoid padding issue
hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
# Missing batch dimension
input_ids = processed_data.get("input_ids")
if is_list_of(input_ids, int):
processed_data["input_ids"] = [input_ids]
return processed_data
return hf_inputs._replace(
hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)
)
def _get_prompt_updates(
self,