[MM] Remove text components from ProcessorInputs (#53093)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -654,18 +654,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
@@ -179,21 +179,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
|
||||
|
||||
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Inkling is not a standard HF processor (no fused text+mm call), so we run
|
||||
# the vendored extractors ourselves and tokenize the text separately.
|
||||
@@ -275,10 +265,8 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
||||
ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
|
||||
|
||||
# Reconcile against the declared media counts only when media is
|
||||
# present. With no media items -- e.g. the base text-only tokenization
|
||||
# probe (``_apply_hf_processor_text_only``), which calls this via
|
||||
# ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers
|
||||
# verbatim; the marker<->item correspondence is enforced later by
|
||||
# present. With no media items, emit the markers verbatim; the
|
||||
# marker<->item correspondence is enforced later by
|
||||
# ``_get_prompt_updates`` once the media features are available.
|
||||
if num_images or num_audios:
|
||||
# Fail clearly on a placeholder/media-count mismatch instead of
|
||||
|
||||
@@ -236,23 +236,13 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override so the base always routes through the text+mm path
|
||||
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
|
||||
# the checkpoint image processor directly with bare PIL images, but it
|
||||
# requires `{"type": "image", "image": PIL}` media dicts that only our
|
||||
# wrapper builds.
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -311,7 +311,6 @@ class MiniMaxM3VLMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
# With ``video_needs_metadata=True`` each video arrives as a
|
||||
@@ -341,7 +340,7 @@ class MiniMaxM3VLMultiModalProcessor(
|
||||
|
||||
# Override the video processor's default do_resize=False (set for a
|
||||
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
|
||||
merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)
|
||||
merged = dict(do_resize=True, **mm_kwargs)
|
||||
data = dict(text=prompt, **mm_data)
|
||||
if video_metadata is not None:
|
||||
data["video_metadata"] = video_metadata
|
||||
|
||||
Reference in New Issue
Block a user