[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
-22
View File
@@ -560,31 +560,11 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _apply_hf_processor_text_only(
self,
prompt_text: str,
tokenization_kwargs: Mapping[str, object],
) -> list[int]:
# Bypass the HF processor and tokenize directly. The HF
# processor expands multimodal placeholders (<|video|>, etc.)
# via get_text_with_replacements, which raises StopIteration
# when the prompt contains placeholders without matching data.
# The text-only path only needs token IDs, so the tokenizer
# alone is sufficient.
processor = self.info.get_hf_processor()
text_inputs = processor.tokenizer([prompt_text], **tokenization_kwargs)
input_ids = text_inputs["input_ids"]
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
return prompt_ids
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
@@ -640,7 +620,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt=dummy_prompt,
mm_data={"images": frames},
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Remap HF key name
@@ -741,7 +720,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt,
mm_data,
patched_mm_kwargs,
tok_kwargs,
)
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.