[MM] Remove text components from ProcessorInputs (#53093)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -456,13 +456,11 @@ return a schema of the tensors outputted by the HF processor that are related to
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
@@ -479,11 +477,6 @@ return a schema of the tensors outputted by the HF processor that are related to
|
||||
return processed_outputs
|
||||
```
|
||||
|
||||
!!! note
|
||||
The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for
|
||||
processing. `mm_kwargs` is used to both initialize and call the huggingface
|
||||
processor, whereas `tok_kwargs` is only used to call the huggingface processor.
|
||||
|
||||
Since `pixel_values` is now a list with one tensor per image, we can override
|
||||
[_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx):
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
prompt = "<|user|>Listen.<|end|>"
|
||||
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
|
||||
@@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test():
|
||||
)
|
||||
|
||||
|
||||
def get_text_token_prompts(
|
||||
def get_token_prompt(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
):
|
||||
) -> list[int]:
|
||||
dummy_inputs = processor.dummy_inputs
|
||||
tokenizer: TokenizerLike = processor.info.get_tokenizer()
|
||||
model_config = processor.info.ctx.model_config
|
||||
@@ -178,21 +178,10 @@ def get_text_token_prompts(
|
||||
mm_options={},
|
||||
)
|
||||
|
||||
text_prompt: str | None
|
||||
token_prompt: list[int]
|
||||
if isinstance(inputs.prompt, list):
|
||||
text_prompt = None
|
||||
token_prompt = inputs.prompt
|
||||
elif isinstance(inputs.prompt, str):
|
||||
text_prompt = inputs.prompt
|
||||
token_prompt = tokenizer.encode(
|
||||
text_prompt,
|
||||
**processor.info.get_default_tok_params().get_encode_kwargs(),
|
||||
)
|
||||
else:
|
||||
if not isinstance(inputs.prompt, list):
|
||||
raise TypeError(type(inputs.prompt))
|
||||
|
||||
return text_prompt, token_prompt
|
||||
return inputs.prompt
|
||||
|
||||
|
||||
def random_vision_chunk(
|
||||
@@ -358,7 +347,7 @@ def _test_processing_correctness_one(
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
|
||||
token_prompt = get_token_prompt(baseline_processor, mm_data)
|
||||
mm_items = baseline_processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
@@ -381,55 +370,10 @@ def _test_processing_correctness_one(
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
f"{token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
if text_prompt is not None:
|
||||
baseline_text_result = baseline_processor(
|
||||
text_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
cached_text_result = cached_processor(
|
||||
text_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_text_result,
|
||||
cached_text_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_text_result,
|
||||
baseline_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
cached_text_result,
|
||||
cached_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{text_prompt=}, {token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
|
||||
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
@@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
@@ -11,6 +11,7 @@ from pqdm.threads import pqdm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
info = processor.info
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
@@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
|
||||
@@ -35,12 +35,10 @@ from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
del add_special_tokens
|
||||
def encode(self, text, **kwargs):
|
||||
return [ord(char) for char in text]
|
||||
|
||||
def decode(self, token_ids, **kwargs):
|
||||
del kwargs
|
||||
return "".join(chr(token_id) for token_id in token_ids)
|
||||
|
||||
def batch_decode(self, batch_token_ids, **kwargs):
|
||||
|
||||
@@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
|
||||
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
|
||||
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
|
||||
|
||||
assert (
|
||||
processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})
|
||||
is False
|
||||
)
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
resolved = prompt_update.resolve(0)
|
||||
content = resolved.content
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from ....utils import create_new_process_for_each_test
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import dummy_hf_overrides
|
||||
from .test_common import get_model_ids_to_test, get_text_token_prompts
|
||||
from .test_common import get_model_ids_to_test, get_token_prompt
|
||||
|
||||
ImageInput = list[Image.Image]
|
||||
VideoInput: TypeAlias = (
|
||||
@@ -107,10 +107,10 @@ def create_batched_mm_kwargs(
|
||||
}
|
||||
|
||||
# video metadata will be added back to the resized video data here.
|
||||
text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
|
||||
token_prompt = get_token_prompt(processor, resized_mm_data)
|
||||
|
||||
mm_kwargs = processor(
|
||||
prompt=token_prompt if text_prompt is None else text_prompt,
|
||||
prompt=token_prompt,
|
||||
mm_items=processor.info.parse_mm_data(resized_mm_data),
|
||||
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
|
||||
)["mm_kwargs"].require_data()
|
||||
|
||||
@@ -382,7 +382,6 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processor_mm_data = dict(mm_data)
|
||||
audios = processor_mm_data.pop("audios", None)
|
||||
@@ -393,7 +392,6 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=processor_mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if "input_features_mask" in outputs:
|
||||
|
||||
@@ -273,15 +273,6 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
|
||||
class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):
|
||||
"""Multimodal processor for BAGEL model."""
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -472,7 +472,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
# HF processor always adds placeholders even when there's no image
|
||||
@@ -484,7 +483,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -140,7 +140,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
@@ -151,7 +150,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
|
||||
@@ -497,18 +497,8 @@ class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -208,40 +208,17 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
if inputs.mm_data_items:
|
||||
if isinstance(inputs.prompt, str):
|
||||
if len(inputs.prompt) > 0:
|
||||
raise ValueError(
|
||||
"CLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty text prompt."
|
||||
)
|
||||
special_tokens = self.info.get_tokenizer().all_special_ids
|
||||
if all(tok in special_tokens for tok in inputs.prompt):
|
||||
inputs.prompt = []
|
||||
else:
|
||||
special_tokens = self.info.get_tokenizer().all_special_ids
|
||||
if all(tok in special_tokens for tok in inputs.prompt):
|
||||
inputs.prompt = []
|
||||
else:
|
||||
raise ValueError(
|
||||
"CLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty token prompt."
|
||||
)
|
||||
|
||||
# For multi-modal data, the prompt after processing should
|
||||
# only contain the dummy image tokens
|
||||
inputs.tokenization_kwargs = {
|
||||
**inputs.tokenization_kwargs,
|
||||
"add_special_tokens": False,
|
||||
}
|
||||
raise ValueError(
|
||||
"CLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty token prompt."
|
||||
)
|
||||
|
||||
return super().apply(inputs, timing_ctx)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -227,13 +227,11 @@ class Cohere2VisionMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
|
||||
# Ensure num_patches is available for proper tensor splitting
|
||||
|
||||
@@ -1936,9 +1936,9 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
||||
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> str | list[int]:
|
||||
) -> list[int]:
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
@@ -1946,7 +1946,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
):
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
@@ -1959,7 +1958,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
|
||||
@@ -159,14 +159,12 @@ class ColModernVBertMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
text_encoding = tokenizer(
|
||||
prompt,
|
||||
return_tensors="pt",
|
||||
**tok_kwargs,
|
||||
)
|
||||
result = BatchFeature(data=dict(text_encoding))
|
||||
|
||||
@@ -187,15 +185,6 @@ class ColModernVBertMultiModalProcessor(
|
||||
|
||||
return result
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -64,7 +64,6 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
# The ColPali tokenizer_config.json ships with a small default
|
||||
@@ -72,13 +71,12 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
|
||||
# by PaliGemmaProcessor, causing a token-count mismatch.
|
||||
# vLLM enforces its own max_model_len, so we disable HF
|
||||
# truncation to keep all image + text tokens intact.
|
||||
tok_kwargs = dict(tok_kwargs, truncation=False)
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, truncation=False),
|
||||
)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
|
||||
|
||||
@@ -292,7 +292,6 @@ class DeepseekOCRMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
|
||||
@@ -163,7 +163,6 @@ class DeepseekOCR2MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
|
||||
@@ -241,7 +241,6 @@ class DeepseekVL2MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -252,7 +251,6 @@ class DeepseekVL2MultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
processed_outputs["num_patches"] = (
|
||||
|
||||
@@ -1115,7 +1115,6 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# when the prompt is not empty but the multimodal data is empty,
|
||||
# directly invoke the tokenizer.
|
||||
@@ -1151,7 +1150,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
processor_output = self.info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
# Divide the processor_output into two modalities: image and video.
|
||||
|
||||
@@ -233,7 +233,6 @@ class FireRedASR2MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
@@ -246,7 +245,6 @@ class FireRedASR2MultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
|
||||
@@ -445,9 +445,9 @@ class FireRedLIDMultiModalProcessor(
|
||||
):
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> str | list[int]:
|
||||
) -> list[int]:
|
||||
# Dummy encoder prompt for profiling (encoder only processes audio).
|
||||
return [0]
|
||||
|
||||
@@ -456,7 +456,6 @@ class FireRedLIDMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
@@ -469,7 +468,6 @@ class FireRedLIDMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
|
||||
@@ -752,7 +752,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
@@ -765,7 +764,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
|
||||
@@ -617,10 +617,9 @@ class FunAudioChatMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt)])
|
||||
|
||||
audios = mm_data.get("audios", [])
|
||||
if not audios:
|
||||
@@ -679,15 +678,6 @@ class FunAudioChatMultiModalProcessor(
|
||||
|
||||
return BatchFeature({"input_ids": input_ids, **mm_inputs})
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -266,13 +266,11 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
|
||||
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
|
||||
|
||||
@@ -262,7 +262,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# HF Transformers audio processor no longer accepts `audios` key.
|
||||
# We pop `audios` and replace it with `audio` key to suppress
|
||||
@@ -273,7 +272,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
|
||||
if "input_features" in processed_outputs:
|
||||
|
||||
@@ -560,31 +560,11 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
def _apply_hf_processor_text_only(
|
||||
self,
|
||||
prompt_text: str,
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> list[int]:
|
||||
# Bypass the HF processor and tokenize directly. The HF
|
||||
# processor expands multimodal placeholders (<|video|>, etc.)
|
||||
# via get_text_with_replacements, which raises StopIteration
|
||||
# when the prompt contains placeholders without matching data.
|
||||
# The text-only path only needs token IDs, so the tokenizer
|
||||
# alone is sufficient.
|
||||
processor = self.info.get_hf_processor()
|
||||
text_inputs = processor.tokenizer([prompt_text], **tokenization_kwargs)
|
||||
input_ids = text_inputs["input_ids"]
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
(prompt_ids,) = input_ids
|
||||
return prompt_ids
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)
|
||||
@@ -640,7 +620,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt=dummy_prompt,
|
||||
mm_data={"images": frames},
|
||||
mm_kwargs=video_mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Remap HF key name
|
||||
@@ -741,7 +720,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt,
|
||||
mm_data,
|
||||
patched_mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
|
||||
# HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.
|
||||
|
||||
@@ -1581,7 +1581,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
if not mm_data:
|
||||
@@ -1602,7 +1601,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
prompt=prompt,
|
||||
mm_data=prepared_data,
|
||||
mm_kwargs=prepared_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -1631,7 +1629,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
prompt="<|begin_of_video|><|video|><|end_of_video|>",
|
||||
mm_data=video_mm_data,
|
||||
mm_kwargs=video_mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
input_ids = video_outputs.pop("input_ids")
|
||||
if swap_video_frame_tokens:
|
||||
@@ -1659,7 +1656,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if swap_video_frame_tokens:
|
||||
input_ids = processed_outputs["input_ids"]
|
||||
|
||||
@@ -466,15 +466,6 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
|
||||
|
||||
|
||||
class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -755,7 +755,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
prompt: str,
|
||||
mm_data: dict[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Normalize input: handle deprecated key and list conversion.
|
||||
if "audios" in mm_data:
|
||||
@@ -782,7 +781,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Postprocess: rename mask and add chunk counts
|
||||
|
||||
@@ -185,7 +185,6 @@ class GraniteSpeechMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
@@ -198,7 +197,6 @@ class GraniteSpeechMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if "audio" in mm_data:
|
||||
|
||||
@@ -736,7 +736,6 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
# HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.
|
||||
@@ -751,7 +750,7 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn
|
||||
return self.info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
|
||||
@@ -195,7 +195,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
for video_idx, video_arr in enumerate(mm_data.get("videos", [])):
|
||||
if video_arr.dtype != np.uint8:
|
||||
@@ -263,15 +262,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
|
||||
|
||||
return processed_outputs
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
|
||||
)
|
||||
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt = tokenizer.encode(
|
||||
prompt_text,
|
||||
**self.info.default_tok_params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
return ProcessorInputs(
|
||||
prompt=prompt_text,
|
||||
prompt=prompt,
|
||||
mm_data_items=dummy_mm_items,
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs or {},
|
||||
tokenization_kwargs={"truncation": False},
|
||||
)
|
||||
|
||||
def get_dummy_mm_data(
|
||||
@@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
images = mm_data.get("images")
|
||||
videos = mm_data.get("videos")
|
||||
@@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor(
|
||||
|
||||
# Build data dict for HF processor (images/videos only)
|
||||
# NOTE: We pass the prompt as-is without token normalization.
|
||||
# Token expansion is handled by vLLM via _get_prompt_updates since
|
||||
# _hf_processor_applies_updates returns False.
|
||||
# Token expansion is handled by vLLM via _get_prompt_updates.
|
||||
data: dict[str, object] = dict(
|
||||
text=prompt,
|
||||
images=images,
|
||||
@@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor(
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
hf_processor=hf_processor,
|
||||
data=data,
|
||||
kwargs=dict(**mm_kwargs, **tok_kwargs),
|
||||
kwargs=mm_kwargs,
|
||||
)
|
||||
|
||||
return processed_outputs
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
# Match BaseMultiModalProcessor behavior:
|
||||
# - raw multimodal inputs: HF processor applies updates
|
||||
# - embedding inputs: vLLM applies updates
|
||||
return super()._hf_processor_applies_updates(
|
||||
prompt_text,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -244,7 +244,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Text-only input not supported in composite processor
|
||||
if not (images := mm_data.get("images", [])):
|
||||
@@ -260,7 +259,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
|
||||
@@ -337,7 +337,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
videos = mm_data.pop("videos", [])
|
||||
@@ -364,7 +363,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
prompt=hf_processor.image_token,
|
||||
mm_data={"images": image},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
image_pixel_values.append(processed_outputs.pop("pixel_values"))
|
||||
|
||||
@@ -387,7 +385,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
prompt=hf_processor.video_token,
|
||||
mm_data={"videos": video},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
video_pixel_values.append(processed_outputs.pop("pixel_values"))
|
||||
|
||||
@@ -406,7 +403,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
|
||||
prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)
|
||||
prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)
|
||||
text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt")
|
||||
text_outputs = tokenizer(prompt, return_tensors="pt")
|
||||
|
||||
return BatchFeature({**text_outputs, **image_outputs, **video_outputs})
|
||||
|
||||
|
||||
@@ -225,13 +225,11 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
@@ -462,11 +460,8 @@ class InternVLMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
|
||||
|
||||
@@ -59,7 +59,6 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# NOTE: We should reverse the order of the mm_data because the
|
||||
# query prompt is placed after the document prompt in the score
|
||||
@@ -67,7 +66,7 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
# stored in the opposite order (query first, then document).
|
||||
for _, value in mm_data.items():
|
||||
value.reverse()
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -466,7 +466,6 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Run the underlying HF processor on text and image data."""
|
||||
# Text-only input is handled as a special case here.
|
||||
|
||||
@@ -1167,11 +1167,10 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_keye.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -383,11 +383,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_keye.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -231,7 +231,6 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call the HuggingFace processor."""
|
||||
# Convert mm_data format: {'audios': [...]} -> {'audio': ...}
|
||||
@@ -256,7 +255,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -263,11 +263,10 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path because vision chunk
|
||||
# is not considered
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -398,7 +398,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Text-only input not supported in composite processor
|
||||
if not (images := mm_data.get("images", [])):
|
||||
@@ -412,7 +411,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
|
||||
@@ -44,13 +44,11 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: LightOnOCR does not use break/end tokens, so we remove them here.
|
||||
|
||||
@@ -326,13 +326,11 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
|
||||
@@ -326,7 +326,6 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
videos = mm_data.pop("videos", [])
|
||||
@@ -337,7 +336,6 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# LLaVA-OneVision processor doesn't support multiple videos
|
||||
@@ -352,7 +350,6 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data={},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
images = mm_data.pop("images", [])
|
||||
@@ -362,7 +359,6 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
prompt=image_token * len(images),
|
||||
mm_data={"images": images},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
image_outputs = {
|
||||
k: v
|
||||
@@ -378,7 +374,6 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
prompt=video_token,
|
||||
mm_data={"videos": video},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
|
||||
@@ -392,22 +387,6 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
)
|
||||
return BatchFeature(combined_outputs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
base_result = super()._hf_processor_applies_updates(
|
||||
prompt_text=prompt_text,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return base_result and mm_items.get_count("video", strict=False) == 0
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -1552,16 +1552,13 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# The wrapped OV2 processor is a bare custom class without the standard
|
||||
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
|
||||
# fails; overriding this method routes the base class to call us
|
||||
# directly.
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(
|
||||
dict(**mm_kwargs, **tok_kwargs)
|
||||
)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
merged_kwargs.setdefault("return_tensors", "pt")
|
||||
call_kwargs = {
|
||||
k: v
|
||||
@@ -1638,7 +1635,6 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
data = dict(output)
|
||||
return BatchFeature(
|
||||
@@ -1736,7 +1732,6 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
prompt=new_prompt,
|
||||
mm_data=merged_mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
data = dict(output)
|
||||
|
||||
@@ -1803,7 +1798,6 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _rename_codec_outputs_to_video(
|
||||
|
||||
@@ -588,7 +588,6 @@ class MiDashengLMMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
@@ -622,7 +621,6 @@ class MiDashengLMMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -917,7 +917,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.
|
||||
Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses
|
||||
@@ -1000,7 +999,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
|
||||
mm_data = {**mm_data, "videos": converted}
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -436,7 +436,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if (audios := mm_data.get("audios")) is None:
|
||||
return {}
|
||||
@@ -453,7 +452,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
prompts=[self.info.audio_pattern] * len(parsed_audios),
|
||||
mm_data={"audios": [[audio] for audio in parsed_audios]},
|
||||
mm_kwargs={**mm_kwargs, "chunk_input": True},
|
||||
tok_kwargs=tok_kwargs,
|
||||
out_keys={"audio_features", "audio_feature_lens"},
|
||||
)
|
||||
|
||||
@@ -486,11 +484,10 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
return {
|
||||
**super().process_mm_inputs(mm_data, mm_kwargs, tok_kwargs),
|
||||
**self.process_audios(mm_data, mm_kwargs, tok_kwargs),
|
||||
**super().process_mm_inputs(mm_data, mm_kwargs),
|
||||
**self.process_audios(mm_data, mm_kwargs),
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
|
||||
@@ -869,7 +869,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if (images := mm_data.get("images")) is None:
|
||||
return {}
|
||||
@@ -886,7 +885,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompts=[self.info.image_pattern] * len(parsed_images),
|
||||
mm_data={"images": [[image] for image in parsed_images]},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
|
||||
)
|
||||
|
||||
@@ -896,7 +894,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if (videos := mm_data.get("videos")) is None:
|
||||
return {}
|
||||
@@ -918,7 +915,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
**mm_kwargs,
|
||||
"max_slice_nums": self.info.get_video_max_slice_num(),
|
||||
},
|
||||
tok_kwargs=tok_kwargs,
|
||||
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
|
||||
)
|
||||
|
||||
@@ -930,11 +926,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
return {
|
||||
**self.process_images(mm_data, mm_kwargs, tok_kwargs),
|
||||
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
|
||||
**self.process_images(mm_data, mm_kwargs),
|
||||
**self.process_videos(mm_data, mm_kwargs),
|
||||
}
|
||||
|
||||
def _apply_prompt_updates(
|
||||
@@ -993,7 +988,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompts: list[str],
|
||||
mm_data: Mapping[str, Sequence[object]],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
*,
|
||||
out_keys: set[str],
|
||||
) -> dict[str, NestedTensors]:
|
||||
@@ -1003,7 +997,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompt=prompts, # type: ignore
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
else:
|
||||
inputs = defaultdict[str, list[torch.Tensor]](list)
|
||||
@@ -1013,7 +1006,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompt=prompt,
|
||||
mm_data={k: v[i] for k, v in mm_data.items()},
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
for k, v in inputs_one.items():
|
||||
@@ -1027,12 +1019,11 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
|
||||
|
||||
return BatchFeature(
|
||||
{
|
||||
@@ -1041,15 +1032,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
}
|
||||
)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -154,7 +154,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if (images := mm_data.get("images")) is None:
|
||||
return {}
|
||||
@@ -221,7 +220,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if (videos := mm_data.get("videos")) is None:
|
||||
return {}
|
||||
|
||||
@@ -238,13 +238,11 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
|
||||
@@ -594,13 +594,11 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
|
||||
@@ -1128,13 +1128,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
hf_processor.process,
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
tokenizer = hf_processor.tokenizer
|
||||
|
||||
@@ -1948,7 +1948,6 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
@@ -2008,7 +2007,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
video_outputs = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
dict(text=VIDEO_PROMPT, **video_mm_data),
|
||||
dict(**video_mm_kwargs, **tok_kwargs),
|
||||
video_mm_kwargs,
|
||||
)
|
||||
|
||||
input_ids = video_outputs.pop("input_ids")
|
||||
@@ -2058,7 +2057,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
|
||||
@@ -947,11 +947,10 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Moondream3's processor handles images directly rather than exposing a
|
||||
# separate `image_processor`, so keep the cache path on text+MM calls.
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
@cached_property
|
||||
def bos_image_placeholder_tokens(self) -> list[int]:
|
||||
@@ -977,18 +976,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
|
||||
"tilings": MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
}
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
# Moondream3 HF processor does NOT expand placeholder tokens.
|
||||
# vLLM expands BOS + <image> so the whole HF image prefix is marked
|
||||
# bidirectional by the multimodal prefix-LM mask.
|
||||
return False
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
InputProcessingContext,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
@@ -102,6 +103,34 @@ MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = {
|
||||
"enable_time_marker",
|
||||
"mel_config",
|
||||
}
|
||||
MOSS_AUDIO_PLACEHOLDER_TOKENS = (
|
||||
MOSS_AUDIO_BOS_TOKEN,
|
||||
MOSS_AUDIO_TOKEN,
|
||||
MOSS_AUDIO_EOS_TOKEN,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_moss_audio_placeholder_tokens(tokenizer: object) -> None:
|
||||
"""Register the audio placeholder tokens if the tokenizer lacks them.
|
||||
|
||||
The MOSS-Audio hub tokenizer does not define the audio placeholder
|
||||
tokens (the reference HF processor patches them in at runtime). Left
|
||||
as plain text, they merge with adjacent tokens under BPE, so prompts
|
||||
no longer contain a stable token sequence for prompt updates to match.
|
||||
"""
|
||||
convert_tokens_to_ids = getattr(tokenizer, "convert_tokens_to_ids", None)
|
||||
add_tokens = getattr(tokenizer, "add_tokens", None)
|
||||
if convert_tokens_to_ids is None or add_tokens is None:
|
||||
return
|
||||
|
||||
unk_token_id = getattr(tokenizer, "unk_token_id", None)
|
||||
missing_tokens = [
|
||||
token
|
||||
for token in MOSS_AUDIO_PLACEHOLDER_TOKENS
|
||||
if convert_tokens_to_ids(token) in (None, unk_token_id)
|
||||
]
|
||||
if missing_tokens:
|
||||
add_tokens(missing_tokens, special_tokens=True)
|
||||
|
||||
|
||||
class MossAudioAudioInputs(TensorSchema):
|
||||
@@ -1157,6 +1186,11 @@ class MossAudioProcessor:
|
||||
|
||||
|
||||
class MossAudioProcessingInfo(BaseProcessingInfo):
|
||||
def __init__(self, ctx: InputProcessingContext) -> None:
|
||||
super().__init__(ctx)
|
||||
if ctx.tokenizer is not None:
|
||||
_ensure_moss_audio_placeholder_tokens(ctx.tokenizer)
|
||||
|
||||
def get_hf_config(self) -> MossAudioConfig:
|
||||
config = self.ctx.get_hf_config()
|
||||
if isinstance(config, MossAudioConfig):
|
||||
@@ -1306,7 +1340,6 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
@@ -1314,15 +1347,10 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
||||
mm_data["audio"] = audios
|
||||
mm_kwargs = dict(mm_kwargs)
|
||||
processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)
|
||||
tok_kwargs = {
|
||||
key: value
|
||||
for key, value in tok_kwargs.items()
|
||||
if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS
|
||||
}
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**processor_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**tok_kwargs),
|
||||
{},
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -466,33 +466,20 @@ class MossTranscribeDiarizeMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
audios = _get_audios_from_mm_data(mm_data)
|
||||
if not audios:
|
||||
input_ids = tokenizer.encode(
|
||||
prompt,
|
||||
add_special_tokens=tok_kwargs.get("add_special_tokens", False),
|
||||
)
|
||||
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")
|
||||
|
||||
processed = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, audio=audios),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
return _add_vllm_audio_metadata(processed, len(audios))
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return mm_items.get_count("audio", strict=False) > 0
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -262,23 +262,11 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
|
||||
class MuseGlimmerMultiModalProcessor(
|
||||
BaseMultiModalProcessor[MuseGlimmerProcessingInfo]
|
||||
):
|
||||
def _apply_hf_processor_text_only(
|
||||
self,
|
||||
prompt_text: str,
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> list[int]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return tokenizer.encode(
|
||||
prompt_text,
|
||||
**{"add_special_tokens": False, **tokenization_kwargs},
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
tokenizer = processor.tokenizer
|
||||
@@ -366,7 +354,6 @@ class MuseGlimmerMultiModalProcessor(
|
||||
video_pixel_values=video_pixels,
|
||||
video_feature_sizes=torch.tensor(video_sizes),
|
||||
)
|
||||
del tok_kwargs
|
||||
return BatchFeature(data=data, tensor_type=None)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -366,14 +366,13 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,
|
||||
so it chooses this path:
|
||||
`type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`
|
||||
"""
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_image_fields_config(self, hf_inputs: BatchFeature):
|
||||
if self.info.is_dynamic_tiler:
|
||||
@@ -714,10 +713,8 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
if not audio_items:
|
||||
return super().apply(inputs, timing_ctx)
|
||||
|
||||
prompt = inputs.prompt
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
if not isinstance(prompt, str):
|
||||
prompt = tokenizer.decode(prompt, skip_special_tokens=False)
|
||||
prompt = tokenizer.decode(inputs.prompt, skip_special_tokens=False)
|
||||
|
||||
# Inject AUDIO_CONTEXT only after <video> tokens whose video
|
||||
# actually contained an audio stream (preserving video-audio pairing).
|
||||
@@ -733,9 +730,6 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
|
||||
inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
|
||||
if inputs.tokenization_kwargs is None:
|
||||
inputs.tokenization_kwargs = {}
|
||||
|
||||
# Bypass the cached path: the HF processor must receive the
|
||||
# prompt (with injected <so_embedding>) and the audio data
|
||||
# together so it can perform audio-token replacement natively.
|
||||
|
||||
@@ -406,9 +406,9 @@ class NemotronParseMultiModalProcessor(
|
||||
):
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> str | list[int]:
|
||||
) -> list[int]:
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
@@ -416,12 +416,9 @@ class NemotronParseMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
else:
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
tokenizer = hf_processor.tokenizer
|
||||
|
||||
@@ -141,16 +141,6 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo])
|
||||
self.info.get_hf_config().vision_config.spatial_merge_size
|
||||
)(hf_inputs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
"""vLLM이 prompt 업데이트를 처리하도록 False 반환."""
|
||||
return False
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -327,15 +327,6 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(pixel_values=MultiModalFieldConfig.batched("image"))
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -346,7 +346,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
# Avoid warning from HF logger for text-only input
|
||||
@@ -358,7 +357,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
|
||||
@@ -337,7 +337,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
# Avoid warning from HF logger for text-only input
|
||||
@@ -349,7 +348,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
|
||||
|
||||
@@ -249,7 +249,6 @@ class PaddleOCRVLMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
final_mm_kwargs = dict(mm_kwargs or {})
|
||||
@@ -259,7 +258,7 @@ class PaddleOCRVLMultiModalProcessor(
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**final_mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)
|
||||
processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(
|
||||
|
||||
@@ -157,7 +157,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
if not mm_data:
|
||||
@@ -168,7 +167,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -401,13 +401,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
input_ids = processed_outputs["input_ids"]
|
||||
@@ -497,12 +495,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
if len(token_ids) and token_ids[0] == tokenizer.bos_token_id:
|
||||
token_ids = token_ids[1:]
|
||||
text = tokenizer.decode(token_ids)
|
||||
for special_tokens in tokenizer.special_tokens_map.values():
|
||||
if isinstance(special_tokens, str):
|
||||
text = text.replace(f"{special_tokens} ", special_tokens)
|
||||
elif isinstance(special_tokens, list):
|
||||
for special_token in special_tokens:
|
||||
text = text.replace(f"{special_token} ", special_token)
|
||||
for special_token in tokenizer.all_special_tokens:
|
||||
text = text.replace(f"{special_token} ", special_token)
|
||||
# perform hf behavior
|
||||
# https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/64f88b6/processing_phi3_v.py#L407
|
||||
pattern = r"<\|image_\d+\|>"
|
||||
|
||||
@@ -873,7 +873,6 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
@@ -884,9 +883,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
if audio_data := mm_data.get("audios", []):
|
||||
mm_data["audios"] = [(data, sr) for data in audio_data]
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
)
|
||||
processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
num_img_tokens = [
|
||||
|
||||
@@ -147,13 +147,11 @@ class Phi4SiglipMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# The HF processor's tokenizer_image_token() replaces the "<image>"
|
||||
@@ -170,20 +168,6 @@ class Phi4SiglipMultiModalProcessor(
|
||||
|
||||
return processed
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
# The HF processor replaces "<image>" with a single -200 placeholder
|
||||
# but does NOT expand it into N vision-encoder tokens. Since we also
|
||||
# re-tokenize the prompt (see _call_hf_processor), prompt updates are
|
||||
# never applied by the HF processor — vLLM handles the expansion via
|
||||
# _apply_prompt_updates.
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -225,14 +225,12 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
# Avoid padding issue
|
||||
tok_kwargs={**tok_kwargs, "return_tensors": None},
|
||||
dict(**mm_kwargs, return_tensors=None),
|
||||
)
|
||||
|
||||
# Missing batch dimension
|
||||
|
||||
@@ -504,7 +504,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
@@ -552,7 +551,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
input_features = hf_inputs.pop("input_features", None)
|
||||
@@ -887,44 +885,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
),
|
||||
]
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
enable_hf_prompt_update: bool,
|
||||
) -> tuple[list[int], BatchFeature, bool]:
|
||||
"""
|
||||
Qwen2.5-Omni reimplements this function to handle text only.
|
||||
"""
|
||||
if isinstance(prompt, str):
|
||||
if enable_hf_prompt_update:
|
||||
return self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode(prompt)
|
||||
else:
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
|
||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_processed_data, False
|
||||
|
||||
def _apply_hf_processor_mm_only(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
|
||||
@@ -939,11 +903,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
)
|
||||
mm_counts["audio"] -= mm_counts["video"]
|
||||
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
_, mm_processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return mm_processed_data
|
||||
|
||||
@@ -1159,11 +1159,10 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override to use the text path instead of token path to use the
|
||||
# video-specific logic in processing_qwen2_5_vl.py
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -240,7 +240,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# NOTE - we rename audios -> audio in mm data because transformers has
|
||||
# deprecated audios for the qwen2audio processor and will remove
|
||||
@@ -265,7 +264,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -1200,7 +1200,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
@@ -1230,7 +1229,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
# released and Transformers version update:
|
||||
# https://github.com/huggingface/transformers/pull/41473
|
||||
mm_kwargs = dict(mm_kwargs)
|
||||
tok_kwargs = dict(tok_kwargs)
|
||||
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
|
||||
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
|
||||
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
|
||||
@@ -1245,7 +1243,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -1279,7 +1276,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
is_update_applied: bool,
|
||||
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""
|
||||
Qwen3-Omni reimplements this function to handle `use_audio_in_video`.
|
||||
"""
|
||||
|
||||
@@ -1285,7 +1285,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
|
||||
@@ -1370,7 +1369,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
prompt="<|vision_start|><|video_pad|><|vision_end|>",
|
||||
mm_data=video_mm_data,
|
||||
mm_kwargs=video_mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Discard HF output input_ids — we use get_video_repl below
|
||||
@@ -1432,7 +1430,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=non_video_mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Replace each placeholder with pre-computed video tokens.
|
||||
|
||||
@@ -191,40 +191,17 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
if inputs.mm_data_items:
|
||||
if isinstance(inputs.prompt, str):
|
||||
if len(inputs.prompt) > 0:
|
||||
raise ValueError(
|
||||
"SigLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty text prompt."
|
||||
)
|
||||
special_tokens = self.info.get_tokenizer().all_special_ids
|
||||
if all(tok in special_tokens for tok in inputs.prompt):
|
||||
inputs.prompt = []
|
||||
else:
|
||||
special_tokens = self.info.get_tokenizer().all_special_ids
|
||||
if all(tok in special_tokens for tok in inputs.prompt):
|
||||
inputs.prompt = []
|
||||
else:
|
||||
raise ValueError(
|
||||
"SigLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty token prompt."
|
||||
)
|
||||
|
||||
# For multi-modal data, the prompt after processing should
|
||||
# only contain the dummy image tokens
|
||||
inputs.tokenization_kwargs = {
|
||||
**inputs.tokenization_kwargs,
|
||||
"add_special_tokens": False,
|
||||
}
|
||||
raise ValueError(
|
||||
"SigLIP accepts text-only or image-only inputs, not both! "
|
||||
"You must pass an image with an empty token prompt."
|
||||
)
|
||||
|
||||
return super().apply(inputs, timing_ctx)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
|
||||
@@ -350,24 +350,21 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Run the HF processor and unpad inputs."""
|
||||
try:
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
)
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
|
||||
return hf_inputs
|
||||
|
||||
def _unpad_audios(
|
||||
@@ -375,7 +372,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
hf_inputs: "BatchFeature",
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> None:
|
||||
"""Replace the audio fields with each audio processed on its own.
|
||||
|
||||
@@ -395,7 +391,7 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
dict(
|
||||
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
|
||||
),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
for audio in audios
|
||||
]
|
||||
@@ -459,13 +455,13 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask which modality owns each token of the expanded prompt, which is the
|
||||
only marking of the tokens an expansion adds around an item."""
|
||||
if any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
@@ -613,29 +609,26 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
tokenization_kwargs = inputs.tokenization_kwargs
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if not isinstance(prompt, str):
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
prompt = hf_processor.decode(prompt)
|
||||
tokenization_kwargs = {
|
||||
**tokenization_kwargs,
|
||||
"add_special_tokens": False,
|
||||
}
|
||||
prompt = hf_processor.decode(prompt)
|
||||
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
hf_processor_mm_kwargs={
|
||||
**hf_processor_mm_kwargs,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (`_call_hf_processor` disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
@@ -731,45 +724,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
return updates
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
enable_hf_prompt_update: bool,
|
||||
) -> tuple[list[int], "BatchFeature", bool]:
|
||||
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
|
||||
in, whatever `enable_hf_prompt_update` asks for.
|
||||
|
||||
This differs from `super()` only for a text prompt with
|
||||
`enable_hf_prompt_update`, where `super()` would keep the expanded token
|
||||
ids the HF processor returns. Both routes give the same ids, so forcing
|
||||
this one costs an extra processor call and buys the guarantee that a
|
||||
request bypassing the cache is identical to a cached one.
|
||||
|
||||
A text prompt only arrives with `enable_hf_prompt_update` when there is
|
||||
no multi-modal processor cache. Requests carrying pre-computed
|
||||
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
|
||||
here, so they fail validation shortly afterwards.
|
||||
"""
|
||||
if isinstance(prompt, str) and enable_hf_prompt_update:
|
||||
logger.warning_once(
|
||||
"Disabling the multi-modal processor cache is extra slow with the "
|
||||
"Transformers modeling backend: the prompt is still tokenized "
|
||||
"unexpanded and the expansion spliced in, to keep the token ids "
|
||||
"identical to what the cache produces, which costs an extra HF "
|
||||
"processor call per request."
|
||||
)
|
||||
return super()._apply_hf_processor_main(
|
||||
prompt,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
|
||||
def _get_num_image_patches(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -824,13 +778,12 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask for the replacement each placeholder expands to, and record it as
|
||||
per-item fields: its token ids, and the tokens or patches behind them."""
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
hf_inputs.pop("token_type_ids", None)
|
||||
|
||||
@@ -195,7 +195,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Text-only input not supported in composite processor
|
||||
if not mm_data.get("audios", []):
|
||||
@@ -218,16 +217,10 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
|
||||
item_processor_data = dict(**mm_data, audios=audios)
|
||||
|
||||
# some tokenizer kwargs are incompatible with UltravoxProcessor
|
||||
tok_kwargs.pop("add_special_tokens", None)
|
||||
tok_kwargs.pop("padding", None)
|
||||
tok_kwargs.pop("truncation", None)
|
||||
|
||||
output = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=item_processor_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
output["audio_features"] = output.pop("audio_values")
|
||||
|
||||
|
||||
@@ -225,7 +225,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
@@ -234,12 +233,11 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
# MistralCommonVoxtralProcessor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
|
||||
outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
# Avoid padding issue
|
||||
tok_kwargs={**tok_kwargs, "return_tensors": None},
|
||||
dict(**mm_kwargs, return_tensors=None),
|
||||
)
|
||||
|
||||
# Missing batch dimension
|
||||
|
||||
@@ -747,9 +747,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
|
||||
class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> str | list[int]:
|
||||
) -> list[int]:
|
||||
# Strictly speaking, whisper encoder only accept audio features.
|
||||
# We create a dummy encoder prompt here which will be padded to
|
||||
# num_audio_tokens. So that we can create dummy data from this
|
||||
@@ -761,7 +761,6 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
@@ -770,19 +769,10 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
# The HF WhisperProcessor passes **kwargs to both the tokenizer
|
||||
# and the feature extractor. Text-tokenizer kwargs like
|
||||
# `truncation` and `max_length` must be removed when audio data
|
||||
# is present, otherwise the feature extractor interprets
|
||||
# `max_length` as raw audio samples and truncates the audio.
|
||||
tok_kwargs = {
|
||||
k: v for k, v in tok_kwargs.items() if k not in ("truncation", "max_length")
|
||||
}
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if "labels" in processed_outputs:
|
||||
processed_outputs["input_ids"] = processed_outputs.pop("labels")
|
||||
|
||||
@@ -654,18 +654,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
@@ -179,21 +179,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
||||
|
||||
|
||||
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Inkling is not a standard HF processor (no fused text+mm call), so we run
|
||||
# the vendored extractors ourselves and tokenize the text separately.
|
||||
@@ -275,10 +265,8 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])
|
||||
ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
|
||||
|
||||
# Reconcile against the declared media counts only when media is
|
||||
# present. With no media items -- e.g. the base text-only tokenization
|
||||
# probe (``_apply_hf_processor_text_only``), which calls this via
|
||||
# ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers
|
||||
# verbatim; the marker<->item correspondence is enforced later by
|
||||
# present. With no media items, emit the markers verbatim; the
|
||||
# marker<->item correspondence is enforced later by
|
||||
# ``_get_prompt_updates`` once the media features are available.
|
||||
if num_images or num_audios:
|
||||
# Fail clearly on a placeholder/media-count mismatch instead of
|
||||
|
||||
@@ -236,23 +236,13 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# Override so the base always routes through the text+mm path
|
||||
# (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls
|
||||
# the checkpoint image processor directly with bare PIL images, but it
|
||||
# requires `{"type": "image", "image": PIL}` media dicts that only our
|
||||
# wrapper builds.
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
return False
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -311,7 +311,6 @@ class MiniMaxM3VLMultiModalProcessor(
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
# With ``video_needs_metadata=True`` each video arrives as a
|
||||
@@ -341,7 +340,7 @@ class MiniMaxM3VLMultiModalProcessor(
|
||||
|
||||
# Override the video processor's default do_resize=False (set for a
|
||||
# pre-resized pipeline) to True for vLLM's raw-frame inputs.
|
||||
merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)
|
||||
merged = dict(do_resize=True, **mm_kwargs)
|
||||
data = dict(text=prompt, **mm_data)
|
||||
if video_metadata is not None:
|
||||
data["video_metadata"] = video_metadata
|
||||
|
||||
@@ -83,12 +83,22 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
|
||||
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
|
||||
|
||||
tokenization_kwargs = {"truncation": False}
|
||||
tokenizer = self.info.ctx.tokenizer
|
||||
dummy_prompt: list[int]
|
||||
if tokenizer is None:
|
||||
# Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only
|
||||
# accept embeddings and have an empty dummy text, so there are no
|
||||
# prompt tokens.
|
||||
dummy_prompt = []
|
||||
else:
|
||||
dummy_prompt = tokenizer.encode(
|
||||
dummy_text,
|
||||
**self.info.default_tok_params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
return ProcessorInputs(
|
||||
prompt=dummy_text,
|
||||
prompt=dummy_prompt,
|
||||
mm_data_items=dummy_mm_items,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
def _get_dummy_audios(
|
||||
|
||||
@@ -17,11 +17,10 @@ class ProcessorInputs:
|
||||
[`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
|
||||
"""
|
||||
|
||||
prompt: str | list[int]
|
||||
prompt: list[int]
|
||||
mm_data_items: MultiModalDataItems
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def get_mm_hashes(
|
||||
self,
|
||||
|
||||
@@ -38,12 +38,7 @@ from ..inputs import (
|
||||
MultiModalKwargsOptionalItems,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from ..parse import (
|
||||
DictEmbeddingItems,
|
||||
EmbeddingItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalUUIDItems,
|
||||
)
|
||||
from ..parse import MultiModalDataItems, MultiModalUUIDItems
|
||||
from .context import BaseProcessingInfo, TimingContext
|
||||
from .dummy_inputs import BaseDummyInputsBuilder
|
||||
from .inputs import ProcessorInputs
|
||||
@@ -1199,11 +1194,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
prompt: str,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems | None = None,
|
||||
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
|
||||
) -> MultiModalInput:
|
||||
if isinstance(prompt, str):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt = tokenizer.encode(
|
||||
prompt,
|
||||
**self.info.default_tok_params.get_encode_kwargs(),
|
||||
)
|
||||
|
||||
processor_inputs = ProcessorInputs(
|
||||
prompt,
|
||||
mm_items,
|
||||
@@ -1307,7 +1309,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# This refers to the data to be passed to HF processor.
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Call the HF processor on the prompt text and
|
||||
@@ -1316,26 +1317,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
)
|
||||
|
||||
def _hf_processor_applies_updates(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
"""
|
||||
Return whether the HF processor applies prompt updates.
|
||||
|
||||
For most HF processors, this should be `True` when multi-modal
|
||||
data items are passed, but `False` when multi-modal embeddings
|
||||
are passed.
|
||||
"""
|
||||
return not any(
|
||||
isinstance(items, (EmbeddingItems, DictEmbeddingItems))
|
||||
for items in mm_items.values()
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
def _apply_hf_processor_text_mm(
|
||||
@@ -1343,13 +1325,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> tuple[list[int], BatchFeature, bool]:
|
||||
) -> tuple[list[int], BatchFeature]:
|
||||
"""
|
||||
Apply the HF processor on the prompt text and multi-modal data
|
||||
together.
|
||||
|
||||
In addition, return whether prompt updates have been applied.
|
||||
"""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
@@ -1360,7 +1339,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt=prompt_text,
|
||||
mm_data=processor_data,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
tok_kwargs=tokenization_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
@@ -1370,35 +1348,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
is_update_applied = self._hf_processor_applies_updates(
|
||||
prompt_text=prompt_text,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return prompt_ids, processed_data, is_update_applied
|
||||
|
||||
def _apply_hf_processor_text_only(
|
||||
self,
|
||||
prompt_text: str,
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> list[int]:
|
||||
"""
|
||||
Apply the HF processor on the prompt text only.
|
||||
|
||||
Since HF processor requires that text and multi-modal items
|
||||
correspond to each other, we create dummy multi-modal items
|
||||
to go along with the text.
|
||||
"""
|
||||
prompt_ids, _, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt_text,
|
||||
mm_items=MultiModalDataItems({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return prompt_ids
|
||||
return prompt_ids, processed_data
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
@@ -1410,10 +1360,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
Most HF processors accept prompt text but not prompt tokens.
|
||||
If the HF processor adds or removes tokens that are not related to
|
||||
multi-modal data, you should override this method so it is consistent
|
||||
with the output of
|
||||
[`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
|
||||
on the
|
||||
corresponding text.
|
||||
with the tokenization of the corresponding text.
|
||||
"""
|
||||
return prompt_tokens
|
||||
|
||||
@@ -1421,7 +1368,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Apply the HF processor on the multi-modal data only.
|
||||
@@ -1435,11 +1381,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
_, mm_processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return mm_processed_data
|
||||
@@ -1455,7 +1400,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
),
|
||||
processor_data,
|
||||
dict(**hf_processor_mm_kwargs, **tokenization_kwargs),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
@@ -1463,41 +1408,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
enable_hf_prompt_update: bool,
|
||||
) -> tuple[list[int], BatchFeature, bool]:
|
||||
"""
|
||||
Apply the HF processor on the prompt text and multi-modal data.
|
||||
Apply the HF processor on the prompt tokens and multi-modal data.
|
||||
|
||||
In addition, return whether prompt updates have been applied
|
||||
(for most HF processors, this should be `True`).
|
||||
|
||||
Note:
|
||||
If `enable_hf_prompt_update=False`, we use HF processor
|
||||
to perform prompt updates if available; HF processor requires
|
||||
that the prompt corresponds to multi-modal items.
|
||||
(`False` here, since for token prompts the updates are normally
|
||||
applied separately via `_maybe_apply_prompt_updates`).
|
||||
"""
|
||||
if isinstance(prompt, str):
|
||||
if enable_hf_prompt_update:
|
||||
return self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
|
||||
else:
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
|
||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_processed_data, False
|
||||
@@ -1615,8 +1541,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt=inputs.prompt,
|
||||
mm_items=inputs.mm_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=inputs.tokenization_kwargs,
|
||||
enable_hf_prompt_update=True,
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -1687,8 +1611,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt=inputs.prompt,
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=inputs.tokenization_kwargs,
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -1938,7 +1860,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
is_update_applied,
|
||||
) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
# NOTE: tokenization_kwargs are not required to init processor
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
mm_items=inputs.mm_data_items,
|
||||
@@ -1967,9 +1888,9 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
@abstractmethod
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> str | list[int]:
|
||||
) -> list[int]:
|
||||
"""
|
||||
Create input prompt for the encoder. HF processor will be applied on
|
||||
this prompt during profiling and generation.
|
||||
@@ -1986,7 +1907,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
def _get_enc_dec_inputs(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
encoder_inputs: MultiModalInput,
|
||||
):
|
||||
@@ -2028,7 +1949,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
inputs.mm_data_items,
|
||||
inputs.mm_uuid_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=inputs.tokenization_kwargs,
|
||||
)
|
||||
|
||||
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
|
||||
|
||||
Reference in New Issue
Block a user