[MM] Remove text components from ProcessorInputs (#53093)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
|
||||
)
|
||||
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt = tokenizer.encode(
|
||||
prompt_text,
|
||||
**self.info.default_tok_params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
return ProcessorInputs(
|
||||
prompt=prompt_text,
|
||||
prompt=prompt,
|
||||
mm_data_items=dummy_mm_items,
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs or {},
|
||||
tokenization_kwargs={"truncation": False},
|
||||
)
|
||||
|
||||
def get_dummy_mm_data(
|
||||
@@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
images = mm_data.get("images")
|
||||
videos = mm_data.get("videos")
|
||||
@@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor(
|
||||
|
||||
# Build data dict for HF processor (images/videos only)
|
||||
# NOTE: We pass the prompt as-is without token normalization.
|
||||
# Token expansion is handled by vLLM via _get_prompt_updates since
|
||||
# _hf_processor_applies_updates returns False.
|
||||
# Token expansion is handled by vLLM via _get_prompt_updates.
|
||||
data: dict[str, object] = dict(
|
||||
text=prompt,
|
||||
images=images,
|
||||
@@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor(
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
hf_processor=hf_processor,
|
||||
data=data,
|
||||
kwargs=dict(**mm_kwargs, **tok_kwargs),
|
||||
kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
return processed_outputs
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
# Match BaseMultiModalProcessor behavior:
|
||||
# - raw multimodal inputs: HF processor applies updates
|
||||
# - embedding inputs: vLLM applies updates
|
||||
return super()._hf_processor_applies_updates(
|
||||
prompt_text,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
Reference in New Issue
Block a user