[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
-7
View File
@@ -456,13 +456,11 @@ return a schema of the tensors outputted by the HF processor that are related to
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values = processed_outputs.get("pixel_values")
@@ -479,11 +477,6 @@ return a schema of the tensors outputted by the HF processor that are related to
return processed_outputs
```
!!! note
The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for
processing. `mm_kwargs` is used to both initialize and call the huggingface
processor, whereas `tok_kwargs` is only used to call the huggingface processor.
Since `pixel_values` is now a list with one tensor per image, we can override
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
@@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx):
mm_data = {"audio": [audio_1, audio_2]}
prompt = "<|user|>Listen.<|end|>"
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
processed = processor._call_hf_processor(prompt, mm_data, {})
chunk_counts = processed["chunk_counts"]
@@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test():
)
def get_text_token_prompts(
def get_token_prompt(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
):
) -> list[int]:
dummy_inputs = processor.dummy_inputs
tokenizer: TokenizerLike = processor.info.get_tokenizer()
model_config = processor.info.ctx.model_config
@@ -178,21 +178,10 @@ def get_text_token_prompts(
mm_options={},
)
text_prompt: str | None
token_prompt: list[int]
if isinstance(inputs.prompt, list):
text_prompt = None
token_prompt = inputs.prompt
elif isinstance(inputs.prompt, str):
text_prompt = inputs.prompt
token_prompt = tokenizer.encode(
text_prompt,
**processor.info.get_default_tok_params().get_encode_kwargs(),
)
else:
if not isinstance(inputs.prompt, list):
raise TypeError(type(inputs.prompt))
return text_prompt, token_prompt
return inputs.prompt
def random_vision_chunk(
@@ -358,7 +347,7 @@ def _test_processing_correctness_one(
):
model_type = model_config.hf_config.model_type
text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
token_prompt = get_token_prompt(baseline_processor, mm_data)
mm_items = baseline_processor.info.parse_mm_data(mm_data)
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
@@ -381,55 +370,10 @@ def _test_processing_correctness_one(
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
f"{token_prompt=}, {mm_data=})"
),
)
if text_prompt is not None:
baseline_text_result = baseline_processor(
text_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_text_result = cached_processor(
text_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
baseline_text_result,
cached_text_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
_assert_inputs_equal(
baseline_text_result,
baseline_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
_assert_inputs_equal(
cached_text_result,
cached_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
),
)
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.tokenizers.hf import maybe_make_thread_pool
from ...utils import build_model_context
@@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
image_ratios = [
(171, 152),
@@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.tokenizers.hf import maybe_make_thread_pool
from ...utils import build_model_context
@@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
info = processor.info
seen_aspect_ratios = set[float]()
@@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
image_ratios = [
(171, 152),
@@ -35,12 +35,10 @@ from vllm.sequence import IntermediateTensors
class _Tokenizer:
def encode(self, text, add_special_tokens=False):
del add_special_tokens
def encode(self, text, **kwargs):
return [ord(char) for char in text]
def decode(self, token_ids, **kwargs):
del kwargs
return "".join(chr(token_id) for token_id in token_ids)
def batch_decode(self, batch_token_ids, **kwargs):
@@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
assert (
processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})
is False
)
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
resolved = prompt_update.resolve(0)
content = resolved.content
@@ -37,7 +37,7 @@ from vllm.utils.torch_utils import set_default_torch_dtype
from ....utils import create_new_process_for_each_test
from ...registry import HF_EXAMPLE_MODELS
from ...utils import dummy_hf_overrides
from .test_common import get_model_ids_to_test, get_text_token_prompts
from .test_common import get_model_ids_to_test, get_token_prompt
ImageInput = list[Image.Image]
VideoInput: TypeAlias = (
@@ -107,10 +107,10 @@ def create_batched_mm_kwargs(
}
# video metadata will be added back to the resized video data here.
text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
token_prompt = get_token_prompt(processor, resized_mm_data)
mm_kwargs = processor(
prompt=token_prompt if text_prompt is None else text_prompt,
prompt=token_prompt,
mm_items=processor.info.parse_mm_data(resized_mm_data),
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
)["mm_kwargs"].require_data()
@@ -382,7 +382,6 @@ class AudioFlamingo3MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processor_mm_data = dict(mm_data)
audios = processor_mm_data.pop("audios", None)
@@ -393,7 +392,6 @@ class AudioFlamingo3MultiModalProcessor(
prompt=prompt,
mm_data=processor_mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "input_features_mask" in outputs:
-9
View File
@@ -273,15 +273,6 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):
"""Multimodal processor for BAGEL model."""
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-2
View File
@@ -472,7 +472,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# HF processor always adds placeholders even when there's no image
@@ -484,7 +483,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
-2
View File
@@ -140,7 +140,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
@@ -151,7 +150,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _apply_hf_processor_tokens_only(
+1 -11
View File
@@ -497,18 +497,8 @@ class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+7 -30
View File
@@ -208,40 +208,17 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
timing_ctx: TimingContext,
) -> MultiModalInput:
if inputs.mm_data_items:
if isinstance(inputs.prompt, str):
if len(inputs.prompt) > 0:
raise ValueError(
"CLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty text prompt."
)
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
raise ValueError(
"CLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
# For multi-modal data, the prompt after processing should
# only contain the dummy image tokens
inputs.tokenization_kwargs = {
**inputs.tokenization_kwargs,
"add_special_tokens": False,
}
raise ValueError(
"CLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
return super().apply(inputs, timing_ctx)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -227,13 +227,11 @@ class Cohere2VisionMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
# Ensure num_patches is available for proper tensor splitting
+2 -4
View File
@@ -1936,9 +1936,9 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
return [0]
def _call_hf_processor(
@@ -1946,7 +1946,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
):
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -1959,7 +1958,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
@@ -159,14 +159,12 @@ class ColModernVBertMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
text_encoding = tokenizer(
prompt,
return_tensors="pt",
**tok_kwargs,
)
result = BatchFeature(data=dict(text_encoding))
@@ -187,15 +185,6 @@ class ColModernVBertMultiModalProcessor(
return result
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+6 -8
View File
@@ -64,7 +64,6 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
# The ColPali tokenizer_config.json ships with a small default
@@ -72,13 +71,12 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
# by PaliGemmaProcessor, causing a token-count mismatch.
# vLLM enforces its own max_model_len, so we disable HF
# truncation to keep all image + text tokens intact.
tok_kwargs = dict(tok_kwargs, truncation=False)
return super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, truncation=False),
)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
@@ -292,7 +292,6 @@ class DeepseekOCRMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = self.info.ctx.call_hf_processor(
@@ -163,7 +163,6 @@ class DeepseekOCR2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = self.info.ctx.call_hf_processor(
@@ -241,7 +241,6 @@ class DeepseekVL2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
tokenizer = self.info.get_tokenizer()
@@ -252,7 +251,6 @@ class DeepseekVL2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
processed_outputs["num_patches"] = (
+1 -2
View File
@@ -1115,7 +1115,6 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# when the prompt is not empty but the multimodal data is empty,
# directly invoke the tokenizer.
@@ -1151,7 +1150,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
processor_output = self.info.ctx.call_hf_processor(
hf_processor,
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
# Divide the processor_output into two modalities: image and video.
@@ -233,7 +233,6 @@ class FireRedASR2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -246,7 +245,6 @@ class FireRedASR2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
+2 -4
View File
@@ -445,9 +445,9 @@ class FireRedLIDMultiModalProcessor(
):
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
# Dummy encoder prompt for profiling (encoder only processes audio).
return [0]
@@ -456,7 +456,6 @@ class FireRedLIDMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -469,7 +468,6 @@ class FireRedLIDMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
-2
View File
@@ -752,7 +752,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -765,7 +764,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
+1 -11
View File
@@ -617,10 +617,9 @@ class FunAudioChatMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
input_ids = torch.tensor([tokenizer.encode(prompt)])
audios = mm_data.get("audios", [])
if not audios:
@@ -679,15 +678,6 @@ class FunAudioChatMultiModalProcessor(
return BatchFeature({"input_ids": input_ids, **mm_inputs})
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
-2
View File
@@ -266,13 +266,11 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
-2
View File
@@ -262,7 +262,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# HF Transformers audio processor no longer accepts `audios` key.
# We pop `audios` and replace it with `audio` key to suppress
@@ -273,7 +272,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
if "input_features" in processed_outputs:
-22
View File
@@ -560,31 +560,11 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def _apply_hf_processor_text_only(
self,
prompt_text: str,
tokenization_kwargs: Mapping[str, object],
) -> list[int]:
# Bypass the HF processor and tokenize directly. The HF
# processor expands multimodal placeholders (<|video|>, etc.)
# via get_text_with_replacements, which raises StopIteration
# when the prompt contains placeholders without matching data.
# The text-only path only needs token IDs, so the tokenizer
# alone is sufficient.
processor = self.info.get_hf_processor()
text_inputs = processor.tokenizer([prompt_text], **tokenization_kwargs)
input_ids = text_inputs["input_ids"]
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
return prompt_ids
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
@@ -640,7 +620,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt=dummy_prompt,
mm_data={"images": frames},
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Remap HF key name
@@ -741,7 +720,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt,
mm_data,
patched_mm_kwargs,
tok_kwargs,
)
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
-4
View File
@@ -1581,7 +1581,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
if not mm_data:
@@ -1602,7 +1601,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt=prompt,
mm_data=prepared_data,
mm_kwargs=prepared_kwargs,
tok_kwargs=tok_kwargs,
)
if (
@@ -1631,7 +1629,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt="<|begin_of_video|><|video|><|end_of_video|>",
mm_data=video_mm_data,
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
input_ids = video_outputs.pop("input_ids")
if swap_video_frame_tokens:
@@ -1659,7 +1656,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if swap_video_frame_tokens:
input_ids = processed_outputs["input_ids"]
-9
View File
@@ -466,15 +466,6 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
-2
View File
@@ -755,7 +755,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
prompt: str,
mm_data: dict[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Normalize input: handle deprecated key and list conversion.
if "audios" in mm_data:
@@ -782,7 +781,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Postprocess: rename mask and add chunk counts
@@ -185,7 +185,6 @@ class GraniteSpeechMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -198,7 +197,6 @@ class GraniteSpeechMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "audio" in mm_data:
+1 -2
View File
@@ -736,7 +736,6 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
@@ -751,7 +750,7 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
return self.info.ctx.call_hf_processor(
hf_processor,
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
def _get_prompt_updates(
@@ -195,7 +195,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
if video_arr.dtype != np.uint8:
@@ -263,15 +262,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
return processed_outputs
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
tokenizer = self.info.get_tokenizer()
prompt = tokenizer.encode(
prompt_text,
**self.info.default_tok_params.get_encode_kwargs(),
)
return ProcessorInputs(
prompt=prompt_text,
prompt=prompt,
mm_data_items=dummy_mm_items,
hf_processor_mm_kwargs=mm_processor_kwargs or {},
tokenization_kwargs={"truncation": False},
)
def get_dummy_mm_data(
@@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
images = mm_data.get("images")
videos = mm_data.get("videos")
@@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor(
# Build data dict for HF processor (images/videos only)
# NOTE: We pass the prompt as-is without token normalization.
# Token expansion is handled by vLLM via _get_prompt_updates since
# _hf_processor_applies_updates returns False.
# Token expansion is handled by vLLM via _get_prompt_updates.
data: dict[str, object] = dict(
text=prompt,
images=images,
@@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor(
processed_outputs = self.info.ctx.call_hf_processor(
hf_processor=hf_processor,
data=data,
kwargs=dict(**mm_kwargs, **tok_kwargs),
kwargs=mm_kwargs,
)
return processed_outputs
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# Match BaseMultiModalProcessor behavior:
# - raw multimodal inputs: HF processor applies updates
# - embedding inputs: vLLM applies updates
return super()._hf_processor_applies_updates(
prompt_text,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-2
View File
@@ -244,7 +244,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not (images := mm_data.get("images", [])):
@@ -260,7 +259,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
+1 -4
View File
@@ -337,7 +337,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
@@ -364,7 +363,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt=hf_processor.image_token,
mm_data={"images": image},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
image_pixel_values.append(processed_outputs.pop("pixel_values"))
@@ -387,7 +385,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt=hf_processor.video_token,
mm_data={"videos": video},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
video_pixel_values.append(processed_outputs.pop("pixel_values"))
@@ -406,7 +403,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt")
text_outputs = tokenizer(prompt, return_tensors="pt")
return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
+1 -6
View File
@@ -225,13 +225,11 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
@@ -462,11 +460,8 @@ class InternVLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
+1 -2
View File
@@ -59,7 +59,6 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# NOTE: We should reverse the order of the mm_data because the
# query prompt is placed after the document prompt in the score
@@ -67,7 +66,7 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
# stored in the opposite order (query first, then document).
for _, value in mm_data.items():
value.reverse()
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@MULTIMODAL_REGISTRY.register_processor(
-1
View File
@@ -466,7 +466,6 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Run the underlying HF processor on text and image data."""
# Text-only input is handled as a special case here.
+1 -2
View File
@@ -1167,11 +1167,10 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+1 -2
View File
@@ -383,11 +383,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_keye.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+1 -2
View File
@@ -231,7 +231,6 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HuggingFace processor."""
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
@@ -256,7 +255,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
def _get_mm_fields_config(
+1 -2
View File
@@ -263,11 +263,10 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path because vision chunk
# is not considered
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
-2
View File
@@ -398,7 +398,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not (images := mm_data.get("images", [])):
@@ -412,7 +411,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
-2
View File
@@ -44,13 +44,11 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
-2
View File
@@ -326,13 +326,11 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values = processed_outputs.get("pixel_values")
@@ -326,7 +326,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
videos = mm_data.pop("videos", [])
@@ -337,7 +336,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# LLaVA-OneVision processor doesn't support multiple videos
@@ -352,7 +350,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=prompt,
mm_data={},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
images = mm_data.pop("images", [])
@@ -362,7 +359,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=image_token * len(images),
mm_data={"images": images},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
image_outputs = {
k: v
@@ -378,7 +374,6 @@ class LlavaOnevisionMultiModalProcessor(
prompt=video_token,
mm_data={"videos": video},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
@@ -392,22 +387,6 @@ class LlavaOnevisionMultiModalProcessor(
)
return BatchFeature(combined_outputs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
base_result = super()._hf_processor_applies_updates(
prompt_text=prompt_text,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return base_result and mm_items.get_count("video", strict=False) == 0
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -1552,16 +1552,13 @@ class LlavaOnevision2MultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# The wrapped OV2 processor is a bare custom class without the standard
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
# fails; overriding this method routes the base class to call us
# directly.
hf_processor = self.info.get_hf_processor(**mm_kwargs)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(
dict(**mm_kwargs, **tok_kwargs)
)
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
call_kwargs = {
k: v
@@ -1638,7 +1635,6 @@ class LlavaOnevision2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
tok_kwargs=tok_kwargs,
)
data = dict(output)
return BatchFeature(
@@ -1736,7 +1732,6 @@ class LlavaOnevision2MultiModalProcessor(
prompt=new_prompt,
mm_data=merged_mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
data = dict(output)
@@ -1803,7 +1798,6 @@ class LlavaOnevision2MultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _rename_codec_outputs_to_video(
@@ -588,7 +588,6 @@ class MiDashengLMMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
audios = mm_data.pop("audios", [])
@@ -622,7 +621,6 @@ class MiDashengLMMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
+1 -2
View File
@@ -917,7 +917,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
@@ -1000,7 +999,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
mm_data = {**mm_data, "videos": converted}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
+2 -5
View File
@@ -436,7 +436,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (audios := mm_data.get("audios")) is None:
return {}
@@ -453,7 +452,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
prompts=[self.info.audio_pattern] * len(parsed_audios),
mm_data={"audios": [[audio] for audio in parsed_audios]},
mm_kwargs={**mm_kwargs, "chunk_input": True},
tok_kwargs=tok_kwargs,
out_keys={"audio_features", "audio_feature_lens"},
)
@@ -486,11 +484,10 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
return {
**super().process_mm_inputs(mm_data, mm_kwargs, tok_kwargs),
**self.process_audios(mm_data, mm_kwargs, tok_kwargs),
**super().process_mm_inputs(mm_data, mm_kwargs),
**self.process_audios(mm_data, mm_kwargs),
}
def _get_prompt_updates(
+4 -22
View File
@@ -869,7 +869,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (images := mm_data.get("images")) is None:
return {}
@@ -886,7 +885,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompts=[self.info.image_pattern] * len(parsed_images),
mm_data={"images": [[image] for image in parsed_images]},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
)
@@ -896,7 +894,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (videos := mm_data.get("videos")) is None:
return {}
@@ -918,7 +915,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
**mm_kwargs,
"max_slice_nums": self.info.get_video_max_slice_num(),
},
tok_kwargs=tok_kwargs,
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
)
@@ -930,11 +926,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
return {
**self.process_images(mm_data, mm_kwargs, tok_kwargs),
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
**self.process_images(mm_data, mm_kwargs),
**self.process_videos(mm_data, mm_kwargs),
}
def _apply_prompt_updates(
@@ -993,7 +988,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompts: list[str],
mm_data: Mapping[str, Sequence[object]],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
*,
out_keys: set[str],
) -> dict[str, NestedTensors]:
@@ -1003,7 +997,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt=prompts, # type: ignore
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
else:
inputs = defaultdict[str, list[torch.Tensor]](list)
@@ -1013,7 +1006,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt=prompt,
mm_data={k: v[i] for k, v in mm_data.items()},
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
for k, v in inputs_one.items():
@@ -1027,12 +1019,11 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)
input_ids = torch.tensor([tokenizer.encode(prompt)])
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
return BatchFeature(
{
@@ -1041,15 +1032,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
}
)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -154,7 +154,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (images := mm_data.get("images")) is None:
return {}
@@ -221,7 +220,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> Mapping[str, NestedTensors]:
if (videos := mm_data.get("videos")) is None:
return {}
-2
View File
@@ -238,13 +238,11 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
pixel_values = processed_outputs.get("pixel_values")
-2
View File
@@ -594,13 +594,11 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
processor = self.info.get_hf_processor(**mm_kwargs)
+1 -2
View File
@@ -1128,13 +1128,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self.info.get_hf_processor(**mm_kwargs)
processed_outputs = self.info.ctx.call_hf_processor(
hf_processor.process,
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
tokenizer = hf_processor.tokenizer
+2 -3
View File
@@ -1948,7 +1948,6 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
@@ -2008,7 +2007,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
video_outputs = self.info.ctx.call_hf_processor(
patched_call,
dict(text=VIDEO_PROMPT, **video_mm_data),
dict(**video_mm_kwargs, **tok_kwargs),
video_mm_kwargs,
)
input_ids = video_outputs.pop("input_ids")
@@ -2058,7 +2057,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
processed_outputs = self.info.ctx.call_hf_processor(
patched_call,
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
if (images := mm_data.get("images")) is not None:
+1 -14
View File
@@ -947,11 +947,10 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Moondream3's processor handles images directly rather than exposing a
# separate `image_processor`, so keep the cache path on text+MM calls.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
@cached_property
def bos_image_placeholder_tokens(self) -> list[int]:
@@ -977,18 +976,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
"tilings": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
}
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# Moondream3 HF processor does NOT expand placeholder tokens.
# vLLM expands BOS + <image> so the whole HF image prefix is marked
# bidirectional by the multimodal prefix-LM mask.
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
+35 -7
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
InputProcessingContext,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
@@ -102,6 +103,34 @@ MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = {
"enable_time_marker",
"mel_config",
}
MOSS_AUDIO_PLACEHOLDER_TOKENS = (
MOSS_AUDIO_BOS_TOKEN,
MOSS_AUDIO_TOKEN,
MOSS_AUDIO_EOS_TOKEN,
)
def _ensure_moss_audio_placeholder_tokens(tokenizer: object) -> None:
"""Register the audio placeholder tokens if the tokenizer lacks them.
The MOSS-Audio hub tokenizer does not define the audio placeholder
tokens (the reference HF processor patches them in at runtime). Left
as plain text, they merge with adjacent tokens under BPE, so prompts
no longer contain a stable token sequence for prompt updates to match.
"""
convert_tokens_to_ids = getattr(tokenizer, "convert_tokens_to_ids", None)
add_tokens = getattr(tokenizer, "add_tokens", None)
if convert_tokens_to_ids is None or add_tokens is None:
return
unk_token_id = getattr(tokenizer, "unk_token_id", None)
missing_tokens = [
token
for token in MOSS_AUDIO_PLACEHOLDER_TOKENS
if convert_tokens_to_ids(token) in (None, unk_token_id)
]
if missing_tokens:
add_tokens(missing_tokens, special_tokens=True)
class MossAudioAudioInputs(TensorSchema):
@@ -1157,6 +1186,11 @@ class MossAudioProcessor:
class MossAudioProcessingInfo(BaseProcessingInfo):
def __init__(self, ctx: InputProcessingContext) -> None:
super().__init__(ctx)
if ctx.tokenizer is not None:
_ensure_moss_audio_placeholder_tokens(ctx.tokenizer)
def get_hf_config(self) -> MossAudioConfig:
config = self.ctx.get_hf_config()
if isinstance(config, MossAudioConfig):
@@ -1306,7 +1340,6 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -1314,15 +1347,10 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
mm_data["audio"] = audios
mm_kwargs = dict(mm_kwargs)
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
tok_kwargs = {
key: value
for key, value in tok_kwargs.items()
if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS
}
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**processor_kwargs),
dict(text=prompt, **mm_data),
dict(**tok_kwargs),
{},
)
def _get_mm_fields_config(
@@ -466,33 +466,20 @@ class MossTranscribeDiarizeMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
audios = _get_audios_from_mm_data(mm_data)
if not audios:
input_ids = tokenizer.encode(
prompt,
add_special_tokens=tok_kwargs.get("add_special_tokens", False),
)
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
processed = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, audio=audios),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
return _add_vllm_audio_metadata(processed, len(audios))
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return mm_items.get_count("audio", strict=False) > 0
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -262,23 +262,11 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
class MuseGlimmerMultiModalProcessor(
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
):
def _apply_hf_processor_text_only(
self,
prompt_text: str,
tokenization_kwargs: Mapping[str, object],
) -> list[int]:
tokenizer = self.info.get_tokenizer()
return tokenizer.encode(
prompt_text,
**{"add_special_tokens": False, **tokenization_kwargs},
)
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processor = self.info.get_hf_processor(**mm_kwargs)
tokenizer = processor.tokenizer
@@ -366,7 +354,6 @@ class MuseGlimmerMultiModalProcessor(
video_pixel_values=video_pixels,
video_feature_sizes=torch.tensor(video_sizes),
)
del tok_kwargs
return BatchFeature(data=data, tensor_type=None)
def _get_mm_fields_config(
@@ -366,14 +366,13 @@ class NanoNemotronVLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
so it chooses this path:
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
"""
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_image_fields_config(self, hf_inputs: BatchFeature):
if self.info.is_dynamic_tiler:
@@ -714,10 +713,8 @@ class NanoNemotronVLMultiModalProcessor(
if not audio_items:
return super().apply(inputs, timing_ctx)
prompt = inputs.prompt
tokenizer = self.info.get_tokenizer()
if not isinstance(prompt, str):
prompt = tokenizer.decode(prompt, skip_special_tokens=False)
prompt = tokenizer.decode(inputs.prompt, skip_special_tokens=False)
# Inject AUDIO_CONTEXT only after <video> tokens whose video
# actually contained an audio stream (preserving video-audio pairing).
@@ -733,9 +730,6 @@ class NanoNemotronVLMultiModalProcessor(
inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
if inputs.tokenization_kwargs is None:
inputs.tokenization_kwargs = {}
# Bypass the cached path: the HF processor must receive the
# prompt (with injected <so_embedding>) and the audio data
# together so it can perform audio-token replacement natively.
+3 -6
View File
@@ -406,9 +406,9 @@ class NemotronParseMultiModalProcessor(
):
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
return [0]
def _call_hf_processor(
@@ -416,12 +416,9 @@ class NemotronParseMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
processed_outputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
else:
hf_processor = self.info.get_hf_processor()
tokenizer = hf_processor.tokenizer
-10
View File
@@ -141,16 +141,6 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo])
self.info.get_hf_config().vision_config.spatial_merge_size
)(hf_inputs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
"""vLLM이 prompt 업데이트를 처리하도록 False 반환."""
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-9
View File
@@ -327,15 +327,6 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
) -> Mapping[str, MultiModalFieldConfig]:
return dict(pixel_values=MultiModalFieldConfig.batched("image"))
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
-2
View File
@@ -346,7 +346,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# Avoid warning from HF logger for text-only input
@@ -358,7 +357,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
hf_processor = self.info.get_hf_processor()
-2
View File
@@ -337,7 +337,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
# Avoid warning from HF logger for text-only input
@@ -349,7 +348,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
hf_processor = self.info.get_hf_processor()
+1 -2
View File
@@ -249,7 +249,6 @@ class PaddleOCRVLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
final_mm_kwargs = dict(mm_kwargs or {})
@@ -259,7 +258,7 @@ class PaddleOCRVLMultiModalProcessor(
processed_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**final_mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
-2
View File
@@ -157,7 +157,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
if not mm_data:
@@ -168,7 +167,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
+2 -8
View File
@@ -401,13 +401,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
input_ids = processed_outputs["input_ids"]
@@ -497,12 +495,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
if len(token_ids) and token_ids[0] == tokenizer.bos_token_id:
token_ids = token_ids[1:]
text = tokenizer.decode(token_ids)
for special_tokens in tokenizer.special_tokens_map.values():
if isinstance(special_tokens, str):
text = text.replace(f"{special_tokens} ", special_tokens)
elif isinstance(special_tokens, list):
for special_token in special_tokens:
text = text.replace(f"{special_token} ", special_token)
for special_token in tokenizer.all_special_tokens:
text = text.replace(f"{special_token} ", special_token)
# perform hf behavior
# https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/64f88b6/processing_phi3_v.py#L407
pattern = r"<\|image_\d+\|>"
+1 -4
View File
@@ -873,7 +873,6 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if not mm_data:
prompt_ids = self.info.get_tokenizer().encode(prompt)
@@ -884,9 +883,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
if audio_data := mm_data.get("audios", []):
mm_data["audios"] = [(data, sr) for data in audio_data]
processed_outputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
num_img_tokens = [
-16
View File
@@ -147,13 +147,11 @@ class Phi4SiglipMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# The HF processor's tokenizer_image_token() replaces the "<image>"
@@ -170,20 +168,6 @@ class Phi4SiglipMultiModalProcessor(
return processed
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
# The HF processor replaces "<image>" with a single -200 placeholder
# but does NOT expand it into N vision-encoder tokens. Since we also
# re-tokenize the prompt (see _call_hf_processor), prompt updates are
# never applied by the HF processor — vLLM handles the expansion via
# _apply_prompt_updates.
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
+4 -6
View File
@@ -225,14 +225,12 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
# Avoid padding issue
tok_kwargs={**tok_kwargs, "return_tensors": None},
dict(**mm_kwargs, return_tensors=None),
)
# Missing batch dimension
@@ -504,7 +504,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -552,7 +551,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
input_features = hf_inputs.pop("input_features", None)
@@ -887,44 +885,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
),
]
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], BatchFeature, bool]:
"""
Qwen2.5-Omni reimplements this function to handle text only.
"""
if isinstance(prompt, str):
if enable_hf_prompt_update:
return self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer.encode(prompt)
else:
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return prompt_ids, mm_processed_data, False
def _apply_hf_processor_mm_only(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
@@ -939,11 +903,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
)
mm_counts["audio"] -= mm_counts["video"]
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
_, mm_processed_data = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return mm_processed_data
+1 -2
View File
@@ -1159,11 +1159,10 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override to use the text path instead of token path to use the
# video-specific logic in processing_qwen2_5_vl.py
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_prompt_updates(
self,
@@ -240,7 +240,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# NOTE - we rename audios -> audio in mm data because transformers has
# deprecated audios for the qwen2audio processor and will remove
@@ -265,7 +264,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
def _get_mm_fields_config(
@@ -1200,7 +1200,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -1230,7 +1229,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
# released and Transformers version update:
# https://github.com/huggingface/transformers/pull/41473
mm_kwargs = dict(mm_kwargs)
tok_kwargs = dict(tok_kwargs)
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
@@ -1245,7 +1243,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if (
@@ -1279,7 +1276,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
is_update_applied: bool,
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""
Qwen3-Omni reimplements this function to handle `use_audio_in_video`.
"""
-3
View File
@@ -1285,7 +1285,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
@@ -1370,7 +1369,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
prompt="<|vision_start|><|video_pad|><|vision_end|>",
mm_data=video_mm_data,
mm_kwargs=video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Discard HF output input_ids — we use get_video_repl below
@@ -1432,7 +1430,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
prompt=prompt,
mm_data=mm_data,
mm_kwargs=non_video_mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Replace each placeholder with pre-computed video tokens.
+7 -30
View File
@@ -191,40 +191,17 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
timing_ctx: TimingContext,
) -> MultiModalInput:
if inputs.mm_data_items:
if isinstance(inputs.prompt, str):
if len(inputs.prompt) > 0:
raise ValueError(
"SigLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty text prompt."
)
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
special_tokens = self.info.get_tokenizer().all_special_ids
if all(tok in special_tokens for tok in inputs.prompt):
inputs.prompt = []
else:
raise ValueError(
"SigLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
# For multi-modal data, the prompt after processing should
# only contain the dummy image tokens
inputs.tokenization_kwargs = {
**inputs.tokenization_kwargs,
"add_special_tokens": False,
}
raise ValueError(
"SigLIP accepts text-only or image-only inputs, not both! "
"You must pass an image with an empty token prompt."
)
return super().apply(inputs, timing_ctx)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -350,24 +350,21 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
prompt_ids = self.info.get_tokenizer().encode(prompt)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
return hf_inputs
def _unpad_audios(
@@ -375,7 +372,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
hf_inputs: "BatchFeature",
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> None:
"""Replace the audio fields with each audio processed on its own.
@@ -395,7 +391,7 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
dict(
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
for audio in audios
]
@@ -459,13 +455,13 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
@@ -613,29 +609,26 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
tokenization_kwargs = inputs.tokenization_kwargs
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if not isinstance(prompt, str):
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
prompt = hf_processor.decode(prompt)
tokenization_kwargs = {
**tokenization_kwargs,
"add_special_tokens": False,
}
prompt = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
hf_processor_mm_kwargs={
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (`_call_hf_processor` disables `add_special_tokens`).
"add_special_tokens": False,
},
)
# Use overrides if provided; fallback to data-dependent hashing.
@@ -731,45 +724,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
return updates
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], "BatchFeature", bool]:
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
in, whatever `enable_hf_prompt_update` asks for.
This differs from `super()` only for a text prompt with
`enable_hf_prompt_update`, where `super()` would keep the expanded token
ids the HF processor returns. Both routes give the same ids, so forcing
this one costs an extra processor call and buys the guarantee that a
request bypassing the cache is identical to a cached one.
A text prompt only arrives with `enable_hf_prompt_update` when there is
no multi-modal processor cache. Requests carrying pre-computed
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
here, so they fail validation shortly afterwards.
"""
if isinstance(prompt, str) and enable_hf_prompt_update:
logger.warning_once(
"Disabling the multi-modal processor cache is extra slow with the "
"Transformers modeling backend: the prompt is still tokenized "
"unexpanded and the expansion spliced in, to keep the token ids "
"identical to what the cache produces, which costs an extra HF "
"processor call per request."
)
return super()._apply_hf_processor_main(
prompt,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
enable_hf_prompt_update=False,
)
def _get_num_image_patches(
self,
hf_inputs: "BatchFeature",
@@ -824,13 +778,12 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as
per-item fields: its token ids, and the tokens or patches behind them."""
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None)
-7
View File
@@ -195,7 +195,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Text-only input not supported in composite processor
if not mm_data.get("audios", []):
@@ -218,16 +217,10 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
item_processor_data = dict(**mm_data, audios=audios)
# some tokenizer kwargs are incompatible with UltravoxProcessor
tok_kwargs.pop("add_special_tokens", None)
tok_kwargs.pop("padding", None)
tok_kwargs.pop("truncation", None)
output = super()._call_hf_processor(
prompt=prompt,
mm_data=item_processor_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
output["audio_features"] = output.pop("audio_values")
+4 -6
View File
@@ -225,7 +225,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
@@ -234,12 +233,11 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
# MistralCommonVoxtralProcessor accepts "audio"
mm_data["audio"] = audios
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
# Avoid padding issue
tok_kwargs={**tok_kwargs, "return_tensors": None},
dict(**mm_kwargs, return_tensors=None),
)
# Missing batch dimension
+2 -12
View File
@@ -747,9 +747,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
# Strictly speaking, whisper encoder only accept audio features.
# We create a dummy encoder prompt here which will be padded to
# num_audio_tokens. So that we can create dummy data from this
@@ -761,7 +761,6 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
@@ -770,19 +769,10 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
)
# The HF WhisperProcessor passes **kwargs to both the tokenizer
# and the feature extractor. Text-tokenizer kwargs like
# `truncation` and `max_length` must be removed when audio data
# is present, otherwise the feature extractor interprets
# `max_length` as raw audio samples and truncates the audio.
tok_kwargs = {
k: v for k, v in tok_kwargs.items() if k not in ("truncation", "max_length")
}
processed_outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
if "labels" in processed_outputs:
processed_outputs["input_ids"] = processed_outputs.pop("labels")
+1 -11
View File
@@ -654,18 +654,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config(
self,
+3 -15
View File
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
@@ -179,21 +179,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Inkling is not a standard HF processor (no fused text+mm call), so we run
# the vendored extractors ourselves and tokenize the text separately.
@@ -275,10 +265,8 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
# Reconcile against the declared media counts only when media is
# present. With no media items -- e.g. the base text-only tokenization
# probe (``_apply_hf_processor_text_only``), which calls this via
# ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers
# verbatim; the marker<->item correspondence is enforced later by
# present. With no media items, emit the markers verbatim; the
# marker<->item correspondence is enforced later by
# ``_get_prompt_updates`` once the media features are available.
if num_images or num_audios:
# Fail clearly on a placeholder/media-count mismatch instead of
+1 -11
View File
@@ -236,23 +236,13 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Override so the base always routes through the text+mm path
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
# the checkpoint image processor directly with bare PIL images, but it
# requires `{"type": "image", "image": PIL}` media dicts that only our
# wrapper builds.
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_fields_config(
self,
@@ -311,7 +311,6 @@ class MiniMaxM3VLMultiModalProcessor(
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
# With ``video_needs_metadata=True`` each video arrives as a
@@ -341,7 +340,7 @@ class MiniMaxM3VLMultiModalProcessor(
# Override the video processor's default do_resize=False (set for a
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)
merged = dict(do_resize=True, **mm_kwargs)
data = dict(text=prompt, **mm_data)
if video_metadata is not None:
data["video_metadata"] = video_metadata
+13 -3
View File
@@ -83,12 +83,22 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
tokenization_kwargs = {"truncation": False}
tokenizer = self.info.ctx.tokenizer
dummy_prompt: list[int]
if tokenizer is None:
# Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only
# accept embeddings and have an empty dummy text, so there are no
# prompt tokens.
dummy_prompt = []
else:
dummy_prompt = tokenizer.encode(
dummy_text,
**self.info.default_tok_params.get_encode_kwargs(),
)
return ProcessorInputs(
prompt=dummy_text,
prompt=dummy_prompt,
mm_data_items=dummy_mm_items,
tokenization_kwargs=tokenization_kwargs,
)
def _get_dummy_audios(
+1 -2
View File
@@ -17,11 +17,10 @@ class ProcessorInputs:
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
"""
prompt: str | list[int]
prompt: list[int]
mm_data_items: MultiModalDataItems
mm_uuid_items: MultiModalUUIDItems | None = None
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
def get_mm_hashes(
self,
+23 -103
View File
@@ -38,12 +38,7 @@ from ..inputs import (
MultiModalKwargsOptionalItems,
PlaceholderRange,
)
from ..parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalUUIDItems,
)
from ..parse import MultiModalDataItems, MultiModalUUIDItems
from .context import BaseProcessingInfo, TimingContext
from .dummy_inputs import BaseDummyInputsBuilder
from .inputs import ProcessorInputs
@@ -1199,11 +1194,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
def __call__(
self,
prompt: str,
prompt: str | list[int],
mm_items: MultiModalDataItems,
mm_uuid_items: MultiModalUUIDItems | None = None,
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
) -> MultiModalInput:
if isinstance(prompt, str):
tokenizer = self.info.get_tokenizer()
prompt = tokenizer.encode(
prompt,
**self.info.default_tok_params.get_encode_kwargs(),
)
processor_inputs = ProcessorInputs(
prompt,
mm_items,
@@ -1307,7 +1309,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# This refers to the data to be passed to HF processor.
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Call the HF processor on the prompt text and
@@ -1316,26 +1317,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
dict(**mm_kwargs, **tok_kwargs),
)
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
"""
Return whether the HF processor applies prompt updates.
For most HF processors, this should be `True` when multi-modal
data items are passed, but `False` when multi-modal embeddings
are passed.
"""
return not any(
isinstance(items, (EmbeddingItems, DictEmbeddingItems))
for items in mm_items.values()
mm_kwargs,
)
def _apply_hf_processor_text_mm(
@@ -1343,13 +1325,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> tuple[list[int], BatchFeature, bool]:
) -> tuple[list[int], BatchFeature]:
"""
Apply the HF processor on the prompt text and multi-modal data
together.
In addition, return whether prompt updates have been applied.
"""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
@@ -1360,7 +1339,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt=prompt_text,
mm_data=processor_data,
mm_kwargs=hf_processor_mm_kwargs,
tok_kwargs=tokenization_kwargs,
)
processed_data.update(passthrough_data)
@@ -1370,35 +1348,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
(prompt_ids,) = input_ids
is_update_applied = self._hf_processor_applies_updates(
prompt_text=prompt_text,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return prompt_ids, processed_data, is_update_applied
def _apply_hf_processor_text_only(
self,
prompt_text: str,
tokenization_kwargs: Mapping[str, object],
) -> list[int]:
"""
Apply the HF processor on the prompt text only.
Since HF processor requires that text and multi-modal items
correspond to each other, we create dummy multi-modal items
to go along with the text.
"""
prompt_ids, _, _ = self._apply_hf_processor_text_mm(
prompt_text=prompt_text,
mm_items=MultiModalDataItems({}),
hf_processor_mm_kwargs={},
tokenization_kwargs=tokenization_kwargs,
)
return prompt_ids
return prompt_ids, processed_data
def _apply_hf_processor_tokens_only(
self,
@@ -1410,10 +1360,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
Most HF processors accept prompt text but not prompt tokens.
If the HF processor adds or removes tokens that are not related to
multi-modal data, you should override this method so it is consistent
with the output of
[`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
on the
corresponding text.
with the tokenization of the corresponding text.
"""
return prompt_tokens
@@ -1421,7 +1368,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Apply the HF processor on the multi-modal data only.
@@ -1435,11 +1381,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
mm_counts = mm_items.get_all_counts()
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
_, mm_processed_data = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return mm_processed_data
@@ -1455,7 +1400,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self.info.get_hf_processor(**hf_processor_mm_kwargs),
),
processor_data,
dict(**hf_processor_mm_kwargs, **tokenization_kwargs),
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
@@ -1463,41 +1408,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
def _apply_hf_processor_main(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], BatchFeature, bool]:
"""
Apply the HF processor on the prompt text and multi-modal data.
Apply the HF processor on the prompt tokens and multi-modal data.
In addition, return whether prompt updates have been applied
(for most HF processors, this should be `True`).
Note:
If `enable_hf_prompt_update=False`, we use HF processor
to perform prompt updates if available; HF processor requires
that the prompt corresponds to multi-modal items.
(`False` here, since for token prompts the updates are normally
applied separately via `_maybe_apply_prompt_updates`).
"""
if isinstance(prompt, str):
if enable_hf_prompt_update:
return self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
else:
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return prompt_ids, mm_processed_data, False
@@ -1615,8 +1541,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt=inputs.prompt,
mm_items=inputs.mm_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
tokenization_kwargs=inputs.tokenization_kwargs,
enable_hf_prompt_update=True,
)
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -1687,8 +1611,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt=inputs.prompt,
mm_items=mm_missing_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
tokenization_kwargs=inputs.tokenization_kwargs,
enable_hf_prompt_update=False,
)
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -1938,7 +1860,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
is_update_applied,
) = self._cached_apply_hf_processor(inputs, timing_ctx)
# NOTE: tokenization_kwargs are not required to init processor
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
mm_items=inputs.mm_data_items,
@@ -1967,9 +1888,9 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
@abstractmethod
def create_encoder_prompt(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
) -> str | list[int]:
) -> list[int]:
"""
Create input prompt for the encoder. HF processor will be applied on
this prompt during profiling and generation.
@@ -1986,7 +1907,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
def _get_enc_dec_inputs(
self,
prompt: str | list[int],
prompt: list[int],
mm_items: MultiModalDataItems,
encoder_inputs: MultiModalInput,
):
@@ -2028,7 +1949,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
inputs.mm_data_items,
inputs.mm_uuid_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
tokenization_kwargs=inputs.tokenization_kwargs,
)
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)