[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
@@ -382,7 +382,6 @@ class AudioFlamingo3MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processor_mm_data = dict(mm_data)
audios = processor_mm_data.pop("audios", None)
@@ -393,7 +392,6 @@ class AudioFlamingo3MultiModalProcessor(
prompt=prompt,
mm_data=processor_mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "input_features_mask" in outputs:
-9
View File
@@ -273,15 +273,6 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):
"""Multimodal processor for BAGEL model."""
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-2
View File
@@ -472,7 +472,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# HF processor always adds placeholders even when there's no image
@@ -484,7 +483,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
-2
View File
@@ -140,7 +140,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
@@ -151,7 +150,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _apply_hf_processor_tokens_only(
+1 -11
View File
@@ -497,18 +497,8 @@ class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+7 -30
View File
@@ -208,40 +208,17 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
timing_ctx: TimingContext,
) -> MultiModalInput:
if inputs.mm_data_items:
if isinstance(inputs.prompt, str):
if len(inputs.prompt) > 0:
raise ValueError(
"CLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty text prompt."
)
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
raise ValueError(
"CLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
# For multi-modal data, the prompt after processing should
# only contain the dummy image tokens
inputs.tokenization_kwargs = {
**inputs.tokenization_kwargs,
"add_special_tokens": False,
}
raise ValueError(
"CLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
return super().apply(inputs, timing_ctx)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -227,13 +227,11 @@ class Cohere2VisionMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
# Ensure num_patches is available for proper tensor splitting
+2 -4
View File
@@ -1936,9 +1936,9 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
return [0]
def _call_hf_processor(
@@ -1946,7 +1946,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
):
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -1959,7 +1958,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
@@ -159,14 +159,12 @@ class ColModernVBertMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
text_encoding = tokenizer(
prompt,
return_tensors="pt",
**tok_kwargs,
)
result = BatchFeature(data=dict(text_encoding))
@@ -187,15 +185,6 @@ class ColModernVBertMultiModalProcessor(
return result
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+6 -8
View File
@@ -64,7 +64,6 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
# The ColPali tokenizer_config.json ships with a small default
@@ -72,13 +71,12 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
# by PaliGemmaProcessor, causing a token-count mismatch.
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
tok_kwargs = dict(tok_kwargs, truncation=False)
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, truncation=False),
)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
@@ -292,7 +292,6 @@ class DeepseekOCRMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = self.info.ctx.call_hf_processor(
@@ -163,7 +163,6 @@ class DeepseekOCR2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = self.info.ctx.call_hf_processor(
@@ -241,7 +241,6 @@ class DeepseekVL2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
tokenizer = self.info.get_tokenizer()
@@ -252,7 +251,6 @@ class DeepseekVL2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
processed_outputs["num_patches"] = (
+1 -2
View File
@@ -1115,7 +1115,6 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# when the prompt is not empty but the multimodal data is empty,
# directly invoke the tokenizer.
@@ -1151,7 +1150,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
processor_output = self.info.ctx.call_hf_processor(
hf_processor,
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
# Divide the processor_output into two modalities: image and video.
@@ -233,7 +233,6 @@ class FireRedASR2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -246,7 +245,6 @@ class FireRedASR2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
+2 -4
View File
@@ -445,9 +445,9 @@ class FireRedLIDMultiModalProcessor(
):
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
# Dummy encoder prompt for profiling (encoder only processes audio).
return [0]
@@ -456,7 +456,6 @@ class FireRedLIDMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -469,7 +468,6 @@ class FireRedLIDMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
-2
View File
@@ -752,7 +752,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -765,7 +764,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
+1 -11
View File
@@ -617,10 +617,9 @@ class FunAudioChatMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
input_ids = torch.tensor([tokenizer.encode(prompt)])
audios = mm_data.get("audios", [])
if not audios:
@@ -679,15 +678,6 @@ class FunAudioChatMultiModalProcessor(
return BatchFeature({"input_ids": input_ids, **mm_inputs})
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
-2
View File
@@ -266,13 +266,11 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
-2
View File
@@ -262,7 +262,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# HF Transformers audio processor no longer accepts `audios` key.
# We pop `audios` and replace it with `audio` key to suppress
@@ -273,7 +272,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
if "input_features" in processed_outputs:
-22
View File
@@ -560,31 +560,11 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _apply_hf_processor_text_only(
self,
prompt_text: str,
tokenization_kwargs: Mapping[str, object],
) -> list[int]:
# Bypass the HF processor and tokenize directly. The HF
# processor expands multimodal placeholders (<|video|>, etc.)
# via get_text_with_replacements, which raises StopIteration
# when the prompt contains placeholders without matching data.
# The text-only path only needs token IDs, so the tokenizer
# alone is sufficient.
processor = self.info.get_hf_processor()
text_inputs = processor.tokenizer([prompt_text], **tokenization_kwargs)
input_ids = text_inputs["input_ids"]
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
return prompt_ids
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
@@ -640,7 +620,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt=dummy_prompt,
mm_data={"images": frames},
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Remap HF key name
@@ -741,7 +720,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt,
mm_data,
patched_mm_kwargs,
tok_kwargs,
)
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
-4
View File
@@ -1581,7 +1581,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
if not mm_data:
@@ -1602,7 +1601,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt=prompt,
mm_data=prepared_data,
mm_kwargs=prepared_kwargs,
tok_kwargs=tok_kwargs,
)
if (
@@ -1631,7 +1629,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt="<|begin_of_video|><|video|><|end_of_video|>",
mm_data=video_mm_data,
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
input_ids = video_outputs.pop("input_ids")
if swap_video_frame_tokens:
@@ -1659,7 +1656,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if swap_video_frame_tokens:
input_ids = processed_outputs["input_ids"]
-9
View File
@@ -466,15 +466,6 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
-2
View File
@@ -755,7 +755,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
prompt: str,
mm_data: dict[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Normalize input: handle deprecated key and list conversion.
if "audios" in mm_data:
@@ -782,7 +781,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Postprocess: rename mask and add chunk counts
@@ -185,7 +185,6 @@ class GraniteSpeechMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -198,7 +197,6 @@ class GraniteSpeechMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "audio" in mm_data:
+1 -2
View File
@@ -736,7 +736,6 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
@@ -751,7 +750,7 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
return self.info.ctx.call_hf_processor(
hf_processor,
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
def _get_prompt_updates(
@@ -195,7 +195,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
if video_arr.dtype != np.uint8:
@@ -263,15 +262,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
return processed_outputs
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
tokenizer = self.info.get_tokenizer()
prompt = tokenizer.encode(
prompt_text,
**self.info.default_tok_params.get_encode_kwargs(),
)
return ProcessorInputs(
prompt=prompt_text,
prompt=prompt,
mm_data_items=dummy_mm_items,
hf_processor_mm_kwargs=mm_processor_kwargs or {},
tokenization_kwargs={"truncation": False},
)
def get_dummy_mm_data(
@@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
images = mm_data.get("images")
videos = mm_data.get("videos")
@@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor(
# Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates since
# _hf_processor_applies_updates returns False.
# Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict(
text=prompt,
images=images,
@@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor(
processed_outputs = self.info.ctx.call_hf_processor(
hf_processor=hf_processor,
data=data,
kwargs=dict(**mm_kwargs, **tok_kwargs),
kwargs=mm_kwargs,
)
return processed_outputs
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# Match BaseMultiModalProcessor behavior:
# - raw multimodal inputs: HF processor applies updates
# - embedding inputs: vLLM applies updates
return super()._hf_processor_applies_updates(
prompt_text,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-2
View File
@@ -244,7 +244,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not (images := mm_data.get("images", [])):
@@ -260,7 +259,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
+1 -4
View File
@@ -337,7 +337,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
@@ -364,7 +363,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt=hf_processor.image_token,
mm_data={"images": image},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
image_pixel_values.append(processed_outputs.pop("pixel_values"))
@@ -387,7 +385,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt=hf_processor.video_token,
mm_data={"videos": video},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
video_pixel_values.append(processed_outputs.pop("pixel_values"))
@@ -406,7 +403,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt")
text_outputs = tokenizer(prompt, return_tensors="pt")
return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
+1 -6
View File
@@ -225,13 +225,11 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
@@ -462,11 +460,8 @@ class InternVLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
+1 -2
View File
@@ -59,7 +59,6 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# NOTE: We should reverse the order of the mm_data because the
# query prompt is placed after the document prompt in the score
@@ -67,7 +66,7 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
# stored in the opposite order (query first, then document).
for _, value in mm_data.items():
value.reverse()
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@MULTIMODAL_REGISTRY.register_processor(
-1
View File
@@ -466,7 +466,6 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Run the underlying HF processor on text and image data."""
# Text-only input is handled as a special case here.
+1 -2
View File
@@ -1167,11 +1167,10 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+1 -2
View File
@@ -383,11 +383,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+1 -2
View File
@@ -231,7 +231,6 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HuggingFace processor."""
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
@@ -256,7 +255,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
def _get_mm_fields_config(
+1 -2
View File
@@ -263,11 +263,10 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path because vision chunk
# is not considered
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
-2
View File
@@ -398,7 +398,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not (images := mm_data.get("images", [])):
@@ -412,7 +411,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
-2
View File
@@ -44,13 +44,11 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
-2
View File
@@ -326,13 +326,11 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values = processed_outputs.get("pixel_values")
@@ -326,7 +326,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
@@ -337,7 +336,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# LLaVA-OneVision processor doesn't support multiple videos
@@ -352,7 +350,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=prompt,
mm_data={},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
images = mm_data.pop("images", [])
@@ -362,7 +359,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=image_token * len(images),
mm_data={"images": images},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
image_outputs = {
k: v
@@ -378,7 +374,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=video_token,
mm_data={"videos": video},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -392,22 +387,6 @@ class LlavaOnevisionMultiModalProcessor(
)
return BatchFeature(combined_outputs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
base_result = super()._hf_processor_applies_updates(
prompt_text=prompt_text,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return base_result and mm_items.get_count("video", strict=False) == 0
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -1552,16 +1552,13 @@ class LlavaOnevision2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# The wrapped OV2 processor is a bare custom class without the standard
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
# fails; overriding this method routes the base class to call us
# directly.
hf_processor = self.info.get_hf_processor(**mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(
dict(**mm_kwargs, **tok_kwargs)
)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
call_kwargs = {
k: v
@@ -1638,7 +1635,6 @@ class LlavaOnevision2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
tok_kwargs=tok_kwargs,
)
data = dict(output)
return BatchFeature(
@@ -1736,7 +1732,6 @@ class LlavaOnevision2MultiModalProcessor(
prompt=new_prompt,
mm_data=merged_mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
data = dict(output)
@@ -1803,7 +1798,6 @@ class LlavaOnevision2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _rename_codec_outputs_to_video(
@@ -588,7 +588,6 @@ class MiDashengLMMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
audios = mm_data.pop("audios", [])
@@ -622,7 +621,6 @@ class MiDashengLMMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
+1 -2
View File
@@ -917,7 +917,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
@@ -1000,7 +999,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
mm_data = {**mm_data, "videos": converted}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+2 -5
View File
@@ -436,7 +436,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (audios := mm_data.get("audios")) is None:
return {}
@@ -453,7 +452,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
prompts=[self.info.audio_pattern] * len(parsed_audios),
mm_data={"audios": [[audio] for audio in parsed_audios]},
mm_kwargs={**mm_kwargs, "chunk_input": True},
tok_kwargs=tok_kwargs,
out_keys={"audio_features", "audio_feature_lens"},
)
@@ -486,11 +484,10 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
return {
**super().process_mm_inputs(mm_data, mm_kwargs, tok_kwargs),
**self.process_audios(mm_data, mm_kwargs, tok_kwargs),
**super().process_mm_inputs(mm_data, mm_kwargs),
**self.process_audios(mm_data, mm_kwargs),
}
def _get_prompt_updates(
+4 -22
View File
@@ -869,7 +869,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (images := mm_data.get("images")) is None:
return {}
@@ -886,7 +885,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompts=[self.info.image_pattern] * len(parsed_images),
mm_data={"images": [[image] for image in parsed_images]},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
)
@@ -896,7 +894,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (videos := mm_data.get("videos")) is None:
return {}
@@ -918,7 +915,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
**mm_kwargs,
"max_slice_nums": self.info.get_video_max_slice_num(),
},
tok_kwargs=tok_kwargs,
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
)
@@ -930,11 +926,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
return {
**self.process_images(mm_data, mm_kwargs, tok_kwargs),
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
**self.process_images(mm_data, mm_kwargs),
**self.process_videos(mm_data, mm_kwargs),
}
def _apply_prompt_updates(
@@ -993,7 +988,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompts: list[str],
mm_data: Mapping[str, Sequence[object]],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
*,
out_keys: set[str],
) -> dict[str, NestedTensors]:
@@ -1003,7 +997,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt=prompts, # type: ignore
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
else:
inputs = defaultdict[str, list[torch.Tensor]](list)
@@ -1013,7 +1006,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt=prompt,
mm_data={k: v[i] for k, v in mm_data.items()},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
for k, v in inputs_one.items():
@@ -1027,12 +1019,11 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)
input_ids = torch.tensor([tokenizer.encode(prompt)])
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
return BatchFeature(
{
@@ -1041,15 +1032,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
}
)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -154,7 +154,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (images := mm_data.get("images")) is None:
return {}
@@ -221,7 +220,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (videos := mm_data.get("videos")) is None:
return {}
-2
View File
@@ -238,13 +238,11 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values = processed_outputs.get("pixel_values")
-2
View File
@@ -594,13 +594,11 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
processor = self.info.get_hf_processor(**mm_kwargs)
+1 -2
View File
@@ -1128,13 +1128,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
processed_outputs = self.info.ctx.call_hf_processor(
hf_processor.process,
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
tokenizer = hf_processor.tokenizer
+2 -3
View File
@@ -1948,7 +1948,6 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
@@ -2008,7 +2007,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
video_outputs = self.info.ctx.call_hf_processor(
patched_call,
dict(text=VIDEO_PROMPT, **video_mm_data),
dict(**video_mm_kwargs, **tok_kwargs),
video_mm_kwargs,
)
input_ids = video_outputs.pop("input_ids")
@@ -2058,7 +2057,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
processed_outputs = self.info.ctx.call_hf_processor(
patched_call,
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
if (images := mm_data.get("images")) is not None:
+1 -14
View File
@@ -947,11 +947,10 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Moondream3's processor handles images directly rather than exposing a
# separate `image_processor`, so keep the cache path on text+MM calls.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@cached_property
def bos_image_placeholder_tokens(self) -> list[int]:
@@ -977,18 +976,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
"tilings": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
}
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# Moondream3 HF processor does NOT expand placeholder tokens.
# vLLM expands BOS + <image> so the whole HF image prefix is marked
# bidirectional by the multimodal prefix-LM mask.
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+35 -7
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
InputProcessingContext,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
@@ -102,6 +103,34 @@ MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = {
"enable_time_marker",
"mel_config",
}
MOSS_AUDIO_PLACEHOLDER_TOKENS = (
MOSS_AUDIO_BOS_TOKEN,
MOSS_AUDIO_TOKEN,
MOSS_AUDIO_EOS_TOKEN,
)
def _ensure_moss_audio_placeholder_tokens(tokenizer: object) -> None:
"""Register the audio placeholder tokens if the tokenizer lacks them.
The MOSS-Audio hub tokenizer does not define the audio placeholder
tokens (the reference HF processor patches them in at runtime). Left
as plain text, they merge with adjacent tokens under BPE, so prompts
no longer contain a stable token sequence for prompt updates to match.
"""
convert_tokens_to_ids = getattr(tokenizer, "convert_tokens_to_ids", None)
add_tokens = getattr(tokenizer, "add_tokens", None)
if convert_tokens_to_ids is None or add_tokens is None:
return
unk_token_id = getattr(tokenizer, "unk_token_id", None)
missing_tokens = [
token
for token in MOSS_AUDIO_PLACEHOLDER_TOKENS
if convert_tokens_to_ids(token) in (None, unk_token_id)
]
if missing_tokens:
add_tokens(missing_tokens, special_tokens=True)
class MossAudioAudioInputs(TensorSchema):
@@ -1157,6 +1186,11 @@ class MossAudioProcessor:
class MossAudioProcessingInfo(BaseProcessingInfo):
def __init__(self, ctx: InputProcessingContext) -> None:
super().__init__(ctx)
if ctx.tokenizer is not None:
_ensure_moss_audio_placeholder_tokens(ctx.tokenizer)
def get_hf_config(self) -> MossAudioConfig:
config = self.ctx.get_hf_config()
if isinstance(config, MossAudioConfig):
@@ -1306,7 +1340,6 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -1314,15 +1347,10 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
mm_data["audio"] = audios
mm_kwargs = dict(mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
tok_kwargs = {
key: value
for key, value in tok_kwargs.items()
if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS
}
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs),
dict(text=prompt, **mm_data),
dict(**tok_kwargs),
{},
)
def _get_mm_fields_config(
@@ -466,33 +466,20 @@ class MossTranscribeDiarizeMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
audios = _get_audios_from_mm_data(mm_data)
if not audios:
input_ids = tokenizer.encode(
prompt,
add_special_tokens=tok_kwargs.get("add_special_tokens", False),
)
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, audio=audios),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
return _add_vllm_audio_metadata(processed, len(audios))
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return mm_items.get_count("audio", strict=False) > 0
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -262,23 +262,11 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
class MuseGlimmerMultiModalProcessor(
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
):
def _apply_hf_processor_text_only(
self,
prompt_text: str,
tokenization_kwargs: Mapping[str, object],
) -> list[int]:
tokenizer = self.info.get_tokenizer()
return tokenizer.encode(
prompt_text,
**{"add_special_tokens": False, **tokenization_kwargs},
)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processor = self.info.get_hf_processor(**mm_kwargs)
tokenizer = processor.tokenizer
@@ -366,7 +354,6 @@ class MuseGlimmerMultiModalProcessor(
video_pixel_values=video_pixels,
video_feature_sizes=torch.tensor(video_sizes),
)
del tok_kwargs
return BatchFeature(data=data, tensor_type=None)
def _get_mm_fields_config(
@@ -366,14 +366,13 @@ class NanoNemotronVLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
so it chooses this path:
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
"""
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_image_fields_config(self, hf_inputs: BatchFeature):
if self.info.is_dynamic_tiler:
@@ -714,10 +713,8 @@ class NanoNemotronVLMultiModalProcessor(
if not audio_items:
return super().apply(inputs, timing_ctx)
prompt = inputs.prompt
tokenizer = self.info.get_tokenizer()
if not isinstance(prompt, str):
prompt = tokenizer.decode(prompt, skip_special_tokens=False)
prompt = tokenizer.decode(inputs.prompt, skip_special_tokens=False)
# Inject AUDIO_CONTEXT only after <video> tokens whose video
# actually contained an audio stream (preserving video-audio pairing).
@@ -733,9 +730,6 @@ class NanoNemotronVLMultiModalProcessor(
inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
if inputs.tokenization_kwargs is None:
inputs.tokenization_kwargs = {}
# Bypass the cached path: the HF processor must receive the
# prompt (with injected <so_embedding>) and the audio data
# together so it can perform audio-token replacement natively.
+3 -6
View File
@@ -406,9 +406,9 @@ class NemotronParseMultiModalProcessor(
):
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
return [0]
def _call_hf_processor(
@@ -416,12 +416,9 @@ class NemotronParseMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
else:
hf_processor = self.info.get_hf_processor()
tokenizer = hf_processor.tokenizer
-10
View File
@@ -141,16 +141,6 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo])
self.info.get_hf_config().vision_config.spatial_merge_size
)(hf_inputs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
"""vLLM이 prompt 업데이트를 처리하도록 False 반환."""
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-9
View File
@@ -327,15 +327,6 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]:
return dict(pixel_values=MultiModalFieldConfig.batched("image"))
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-2
View File
@@ -346,7 +346,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# Avoid warning from HF logger for text-only input
@@ -358,7 +357,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
hf_processor = self.info.get_hf_processor()
-2
View File
@@ -337,7 +337,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# Avoid warning from HF logger for text-only input
@@ -349,7 +348,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
hf_processor = self.info.get_hf_processor()
+1 -2
View File
@@ -249,7 +249,6 @@ class PaddleOCRVLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
final_mm_kwargs = dict(mm_kwargs or {})
@@ -259,7 +258,7 @@ class PaddleOCRVLMultiModalProcessor(
processed_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**final_mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
-2
View File
@@ -157,7 +157,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
if not mm_data:
@@ -168,7 +167,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
+2 -8
View File
@@ -401,13 +401,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
input_ids = processed_outputs["input_ids"]
@@ -497,12 +495,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
if len(token_ids) and token_ids[0] == tokenizer.bos_token_id:
token_ids = token_ids[1:]
text = tokenizer.decode(token_ids)
for special_tokens in tokenizer.special_tokens_map.values():
if isinstance(special_tokens, str):
text = text.replace(f"{special_tokens} ", special_tokens)
elif isinstance(special_tokens, list):
for special_token in special_tokens:
text = text.replace(f"{special_token} ", special_token)
for special_token in tokenizer.all_special_tokens:
text = text.replace(f"{special_token} ", special_token)
# perform hf behavior
# https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/64f88b6/processing_phi3_v.py#L407
pattern = r"<\|image_\d+\|>"
+1 -4
View File
@@ -873,7 +873,6 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
@@ -884,9 +883,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
if audio_data := mm_data.get("audios", []):
mm_data["audios"] = [(data, sr) for data in audio_data]
processed_outputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
num_img_tokens = [
-16
View File
@@ -147,13 +147,11 @@ class Phi4SiglipMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# The HF processor's tokenizer_image_token() replaces the "<image>"
@@ -170,20 +168,6 @@ class Phi4SiglipMultiModalProcessor(
return processed
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# The HF processor replaces "<image>" with a single -200 placeholder
# but does NOT expand it into N vision-encoder tokens. Since we also
# re-tokenize the prompt (see _call_hf_processor), prompt updates are
# never applied by the HF processor — vLLM handles the expansion via
# _apply_prompt_updates.
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+4 -6
View File
@@ -225,14 +225,12 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
# Avoid padding issue
tok_kwargs={**tok_kwargs, "return_tensors": None},
dict(**mm_kwargs, return_tensors=None),
)
# Missing batch dimension
@@ -504,7 +504,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -552,7 +551,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
input_features = hf_inputs.pop("input_features", None)
@@ -887,44 +885,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
),
]
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], BatchFeature, bool]:
"""
Qwen2.5-Omni reimplements this function to handle text only.
"""
if isinstance(prompt, str):
if enable_hf_prompt_update:
return self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt)
else:
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return prompt_ids, mm_processed_data, False
def _apply_hf_processor_mm_only(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
@@ -939,11 +903,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
)
mm_counts["audio"] -= mm_counts["video"]
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
_, mm_processed_data = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return mm_processed_data
+1 -2
View File
@@ -1159,11 +1159,10 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_qwen2_5_vl.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
@@ -240,7 +240,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# NOTE - we rename audios -> audio in mm data because transformers has
# deprecated audios for the qwen2audio processor and will remove
@@ -265,7 +264,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
@@ -1200,7 +1200,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -1230,7 +1229,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
mm_kwargs = dict(mm_kwargs)
tok_kwargs = dict(tok_kwargs)
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
@@ -1245,7 +1243,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if (
@@ -1279,7 +1276,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
is_update_applied: bool,
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""
Qwen3-Omni reimplements this function to handle `use_audio_in_video`.
"""
-3
View File
@@ -1285,7 +1285,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
@@ -1370,7 +1369,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
prompt="<|vision_start|><|video_pad|><|vision_end|>",
mm_data=video_mm_data,
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Discard HF output input_ids — we use get_video_repl below
@@ -1432,7 +1430,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
prompt=prompt,
mm_data=mm_data,
mm_kwargs=non_video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Replace each placeholder with pre-computed video tokens.
+7 -30
View File
@@ -191,40 +191,17 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
timing_ctx: TimingContext,
) -> MultiModalInput:
if inputs.mm_data_items:
if isinstance(inputs.prompt, str):
if len(inputs.prompt) > 0:
raise ValueError(
"SigLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty text prompt."
)
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
raise ValueError(
"SigLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
# For multi-modal data, the prompt after processing should
# only contain the dummy image tokens
inputs.tokenization_kwargs = {
**inputs.tokenization_kwargs,
"add_special_tokens": False,
}
raise ValueError(
"SigLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
return super().apply(inputs, timing_ctx)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -350,24 +350,21 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
prompt_ids = self.info.get_tokenizer().encode(prompt)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
return hf_inputs
def _unpad_audios(
@@ -375,7 +372,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
hf_inputs: "BatchFeature",
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> None:
"""Replace the audio fields with each audio processed on its own.
@@ -395,7 +391,7 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
dict(
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
for audio in audios
]
@@ -459,13 +455,13 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
@@ -613,29 +609,26 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
tokenization_kwargs = inputs.tokenization_kwargs
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if not isinstance(prompt, str):
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
prompt = hf_processor.decode(prompt)
tokenization_kwargs = {
**tokenization_kwargs,
"add_special_tokens": False,
}
prompt = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
hf_processor_mm_kwargs={
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (`_call_hf_processor` disables `add_special_tokens`).
"add_special_tokens": False,
},
)
# Use overrides if provided; fallback to data-dependent hashing.
@@ -731,45 +724,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
return updates
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], "BatchFeature", bool]:
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
in, whatever `enable_hf_prompt_update` asks for.
This differs from `super()` only for a text prompt with
`enable_hf_prompt_update`, where `super()` would keep the expanded token
ids the HF processor returns. Both routes give the same ids, so forcing
this one costs an extra processor call and buys the guarantee that a
request bypassing the cache is identical to a cached one.
A text prompt only arrives with `enable_hf_prompt_update` when there is
no multi-modal processor cache. Requests carrying pre-computed
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
here, so they fail validation shortly afterwards.
"""
if isinstance(prompt, str) and enable_hf_prompt_update:
logger.warning_once(
"Disabling the multi-modal processor cache is extra slow with the "
"Transformers modeling backend: the prompt is still tokenized "
"unexpanded and the expansion spliced in, to keep the token ids "
"identical to what the cache produces, which costs an extra HF "
"processor call per request."
)
return super()._apply_hf_processor_main(
prompt,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
enable_hf_prompt_update=False,
)
def _get_num_image_patches(
self,
hf_inputs: "BatchFeature",
@@ -824,13 +778,12 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as
per-item fields: its token ids, and the tokens or patches behind them."""
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None)
-7
View File
@@ -195,7 +195,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not mm_data.get("audios", []):
@@ -218,16 +217,10 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
item_processor_data = dict(**mm_data, audios=audios)
# some tokenizer kwargs are incompatible with UltravoxProcessor
tok_kwargs.pop("add_special_tokens", None)
tok_kwargs.pop("padding", None)
tok_kwargs.pop("truncation", None)
output = super()._call_hf_processor(
prompt=prompt,
mm_data=item_processor_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
output["audio_features"] = output.pop("audio_values")
+4 -6
View File
@@ -225,7 +225,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -234,12 +233,11 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
# Avoid padding issue
tok_kwargs={**tok_kwargs, "return_tensors": None},
dict(**mm_kwargs, return_tensors=None),
)
# Missing batch dimension
+2 -12
View File
@@ -747,9 +747,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
# Strictly speaking, whisper encoder only accept audio features.
# We create a dummy encoder prompt here which will be padded to
# num_audio_tokens. So that we can create dummy data from this
@@ -761,7 +761,6 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -770,19 +769,10 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
# The HF WhisperProcessor passes **kwargs to both the tokenizer
# and the feature extractor. Text-tokenizer kwargs like
# `truncation` and `max_length` must be removed when audio data
# is present, otherwise the feature extractor interprets
# `max_length` as raw audio samples and truncates the audio.
tok_kwargs = {
k: v for k, v in tok_kwargs.items() if k not in ("truncation", "max_length")
}
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")