[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
+2 -7
View File
@@ -649,13 +649,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
processor_data["videos"] = raw_videos
return processor_data, passthrough_data
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_mm_fields_config(
self,
+18 -14
View File
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
@@ -135,8 +135,8 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
num_images = mm_counts.get("image", 0)
num_audios = mm_counts.get("audio", 0)
# Use spellings the renderer would emit; tokenization is bypassed in
# _call_hf_processor (we build input_ids directly), so the exact text
# only needs to be a stable per-item marker.
# _apply_hf_processor_main (we build input_ids directly), so the exact
# text only needs to be a stable per-item marker.
return ("<|content_image|>" * num_images) + (
"<|content_audio_input|>" * num_audios
)
@@ -179,17 +179,19 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Inkling is not a standard HF processor (no fused text+mm call), so we run
# the vendored extractors ourselves and tokenize the text separately.
# The MM placeholders in `prompt` are expanded later by the prompt
# updates, so here we emit ONE placeholder id per media item.
processor = self.info.get_hf_processor(**mm_kwargs)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
images = mm_data.get("images") or []
@@ -200,7 +202,7 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
audios = list(cast(Iterable[Any], audios))
prompt_ids = self._tokenize_with_placeholders(
prompt, tokenizer, len(images), len(audios)
prompt_text, tokenizer, len(images), len(audios)
)
data: dict[str, Any] = {"input_ids": [prompt_ids]}
@@ -232,7 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
data["input_audio_features"] = input_audio_features
data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
return BatchFeature(data=data, tensor_type=None)
processed_data = BatchFeature(data=data, tensor_type=None)
processed_data.update(passthrough_data)
return processed_data
def _tokenize_with_placeholders(
self,
@@ -231,19 +231,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):
class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
"""Image-only multi-modal processor for Kimi-K3."""
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override so the base always routes through the text+mm path
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
# the checkpoint image processor directly with bare PIL images, but it
# requires `{"type": "image", "image": PIL}` media dicts that only our
# wrapper builds.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+19 -8
View File
@@ -306,12 +306,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing
class MiniMaxM3VLMultiModalProcessor(
BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]
):
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not mm_data:
return BatchFeature(dict(passthrough_data))
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
mm_data = dict(mm_data)
# With ``video_needs_metadata=True`` each video arrives as a
# ``(frames, metadata)`` tuple. Split the frames back out and forward the
@@ -340,15 +349,17 @@ class MiniMaxM3VLMultiModalProcessor(
# Override the video processor's default do_resize=False (set for a
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **mm_kwargs)
data = dict(text=prompt, **mm_data)
merged = dict(do_resize=True, **hf_processor_mm_kwargs)
data = dict(text=prompt_text, **mm_data)
if video_metadata is not None:
data["video_metadata"] = video_metadata
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
data,
merged,
)
processed_data.update(passthrough_data)
return processed_data
def _get_mm_fields_config(
self,