[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
+1 -11
View File
@@ -654,18 +654,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config(
self,
+3 -15
View File
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
@@ -179,21 +179,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Inkling is not a standard HF processor (no fused text+mm call), so we run
# the vendored extractors ourselves and tokenize the text separately.
@@ -275,10 +265,8 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
# Reconcile against the declared media counts only when media is
# present. With no media items -- e.g. the base text-only tokenization
# probe (``_apply_hf_processor_text_only``), which calls this via
# ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers
# verbatim; the marker<->item correspondence is enforced later by
# present. With no media items, emit the markers verbatim; the
# marker<->item correspondence is enforced later by
# ``_get_prompt_updates`` once the media features are available.
if num_images or num_audios:
# Fail clearly on a placeholder/media-count mismatch instead of
+1 -11
View File
@@ -236,23 +236,13 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override so the base always routes through the text+mm path
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
# the checkpoint image processor directly with bare PIL images, but it
# requires `{"type": "image", "image": PIL}` media dicts that only our
# wrapper builds.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config(
self,
@@ -311,7 +311,6 @@ class MiniMaxM3VLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
# With ``video_needs_metadata=True`` each video arrives as a
@@ -341,7 +340,7 @@ class MiniMaxM3VLMultiModalProcessor(
# Override the video processor's default do_resize=False (set for a
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)
merged = dict(do_resize=True, **mm_kwargs)
data = dict(text=prompt, **mm_data)
if video_metadata is not None:
data["video_metadata"] = video_metadata