[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
@@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
tokenizer = self.info.get_tokenizer()
prompt = tokenizer.encode(
prompt_text,
**self.info.default_tok_params.get_encode_kwargs(),
)
return ProcessorInputs(
prompt=prompt_text,
prompt=prompt,
mm_data_items=dummy_mm_items,
hf_processor_mm_kwargs=mm_processor_kwargs or {},
tokenization_kwargs={"truncation": False},
)
def get_dummy_mm_data(
@@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
images = mm_data.get("images")
videos = mm_data.get("videos")
@@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor(
# Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates since
# _hf_processor_applies_updates returns False.
# Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict(
text=prompt,
images=images,
@@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor(
processed_outputs = self.info.ctx.call_hf_processor(
hf_processor=hf_processor,
data=data,
kwargs=dict(**mm_kwargs, **tok_kwargs),
kwargs=mm_kwargs,
)
return processed_outputs
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# Match BaseMultiModalProcessor behavior:
# - raw multimodal inputs: HF processor applies updates
# - embedding inputs: vLLM applies updates
return super()._hf_processor_applies_updates(
prompt_text,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,