diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index 33d89db75d..3d16e0e91f 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -456,13 +456,11 @@ return a schema of the tensors outputted by the HF processor that are related to prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) pixel_values = processed_outputs.get("pixel_values") @@ -479,11 +477,6 @@ return a schema of the tensors outputted by the HF processor that are related to return processed_outputs ``` - !!! note - The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for - processing. `mm_kwargs` is used to both initialize and call the huggingface - processor, whereas `tok_kwargs` is only used to call the huggingface processor. - Since `pixel_values` is now a list with one tensor per image, we can override [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py index ae0da09e1f..c7dffa90de 100644 --- a/tests/models/multimodal/processing/test_audioflamingo3.py +++ b/tests/models/multimodal/processing/test_audioflamingo3.py @@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx): mm_data = {"audio": [audio_1, audio_2]} prompt = "<|user|>Listen.<|end|>" - processed = processor._call_hf_processor(prompt, mm_data, {}, {}) + processed = processor._call_hf_processor(prompt, mm_data, {}) chunk_counts = processed["chunk_counts"] diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 0754892e4e..ce79ce2435 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test(): ) -def get_text_token_prompts( +def get_token_prompt( processor: BaseMultiModalProcessor, mm_data: MultiModalDataDict, -): +) -> list[int]: dummy_inputs = processor.dummy_inputs tokenizer: TokenizerLike = processor.info.get_tokenizer() model_config = processor.info.ctx.model_config @@ -178,21 +178,10 @@ def get_text_token_prompts( mm_options={}, ) - text_prompt: str | None - token_prompt: list[int] - if isinstance(inputs.prompt, list): - text_prompt = None - token_prompt = inputs.prompt - elif isinstance(inputs.prompt, str): - text_prompt = inputs.prompt - token_prompt = tokenizer.encode( - text_prompt, - **processor.info.get_default_tok_params().get_encode_kwargs(), - ) - else: + if not isinstance(inputs.prompt, list): raise TypeError(type(inputs.prompt)) - return text_prompt, token_prompt + return inputs.prompt def random_vision_chunk( @@ -358,7 +347,7 @@ def _test_processing_correctness_one( ): model_type = model_config.hf_config.model_type - text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data) + token_prompt = get_token_prompt(baseline_processor, mm_data) mm_items = baseline_processor.info.parse_mm_data(mm_data) ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]()) @@ -381,55 +370,10 @@ def _test_processing_correctness_one( msg=( f"Failed ({batch_idx=}, {hit_rate=}, " f"{num_batches=}, {simplify_rate=}, " - f"{text_prompt=}, {token_prompt=}, {mm_data=})" + f"{token_prompt=}, {mm_data=})" ), ) - if text_prompt is not None: - baseline_text_result = baseline_processor( - text_prompt, - mm_items=mm_items, - hf_processor_mm_kwargs={}, - ) - cached_text_result = cached_processor( - text_prompt, - mm_items=mm_items, - hf_processor_mm_kwargs={}, - ) - - _assert_inputs_equal( - baseline_text_result, - cached_text_result, - ignore_mm_keys=ignore_mm_keys, - msg=( - f"Failed ({batch_idx=}, {hit_rate=}, " - f"{num_batches=}, {simplify_rate=}, " - f"{text_prompt=}, {token_prompt=}, {mm_data=})" - ), - ) - - _assert_inputs_equal( - baseline_text_result, - baseline_tokenized_result, - ignore_mm_keys=ignore_mm_keys, - msg=( - f"Failed ({batch_idx=}, {hit_rate=}, " - f"{num_batches=}, {simplify_rate=}, " - f"{text_prompt=}, {token_prompt=}, {mm_data=})" - ), - ) - - _assert_inputs_equal( - cached_text_result, - cached_tokenized_result, - ignore_mm_keys=ignore_mm_keys, - msg=( - f"Failed ({batch_idx=}, {hit_rate=}, " - f"{num_batches=}, {simplify_rate=}, " - f"{text_prompt=}, {token_prompt=}, {mm_data=})" - ), - ) - @pytest.mark.parametrize("model_id", get_model_ids_to_test()) @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0]) diff --git a/tests/models/multimodal/processing/test_llava_next.py b/tests/models/multimodal/processing/test_llava_next.py index b72c1bfd8e..d7eaa35dd8 100644 --- a/tests/models/multimodal/processing/test_llava_next.py +++ b/tests/models/multimodal/processing/test_llava_next.py @@ -11,6 +11,7 @@ from pqdm.threads import pqdm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import ImageSize from vllm.multimodal.processing import BaseMultiModalProcessor +from vllm.tokenizers.hf import maybe_make_thread_pool from ...utils import build_model_context @@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + # Avoid tokenizer already borrowed error + maybe_make_thread_pool(ctx.tokenizer) + + processor = MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) image_ratios = [ (171, 152), @@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + # Avoid tokenizer already borrowed error + maybe_make_thread_pool(ctx.tokenizer) + + processor = MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) seen_aspect_ratios = set[float]() image_sizes = list[ImageSize]() diff --git a/tests/models/multimodal/processing/test_llava_onevision.py b/tests/models/multimodal/processing/test_llava_onevision.py index 2bac464e78..4e8a8103a5 100644 --- a/tests/models/multimodal/processing/test_llava_onevision.py +++ b/tests/models/multimodal/processing/test_llava_onevision.py @@ -11,6 +11,7 @@ from pqdm.threads import pqdm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import ImageSize from vllm.multimodal.processing import BaseMultiModalProcessor +from vllm.tokenizers.hf import maybe_make_thread_pool from ...utils import build_model_context @@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id): mm_processor_kwargs=None, limit_mm_per_prompt={"image": 1}, ) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + # Avoid tokenizer already borrowed error + maybe_make_thread_pool(ctx.tokenizer) + + processor = MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) + info = processor.info seen_aspect_ratios = set[float]() @@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs): mm_processor_kwargs=None, limit_mm_per_prompt={"image": num_imgs}, ) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + # Avoid tokenizer already borrowed error + maybe_make_thread_pool(ctx.tokenizer) + + processor = MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) image_ratios = [ (171, 152), diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py index 8a18d90f56..87e7079a81 100644 --- a/tests/models/multimodal/processing/test_moss_audio.py +++ b/tests/models/multimodal/processing/test_moss_audio.py @@ -35,12 +35,10 @@ from vllm.sequence import IntermediateTensors class _Tokenizer: - def encode(self, text, add_special_tokens=False): - del add_special_tokens + def encode(self, text, **kwargs): return [ord(char) for char in text] def decode(self, token_ids, **kwargs): - del kwargs return "".join(chr(token_id) for token_id in token_ids) def batch_decode(self, batch_token_ids, **kwargs): diff --git a/tests/models/multimodal/processing/test_openvla.py b/tests/models/multimodal/processing/test_openvla.py index b9ed02000d..992608f893 100644 --- a/tests/models/multimodal/processing/test_openvla.py +++ b/tests/models/multimodal/processing/test_openvla.py @@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None: image = Image.new("RGB", (640, 480), color=(255, 255, 255)) mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])}) - assert ( - processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {}) - is False - ) - prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0] resolved = prompt_update.resolve(0) content = resolved.content diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index 12c5071978..87cf65b8f9 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -37,7 +37,7 @@ from vllm.utils.torch_utils import set_default_torch_dtype from ....utils import create_new_process_for_each_test from ...registry import HF_EXAMPLE_MODELS from ...utils import dummy_hf_overrides -from .test_common import get_model_ids_to_test, get_text_token_prompts +from .test_common import get_model_ids_to_test, get_token_prompt ImageInput = list[Image.Image] VideoInput: TypeAlias = ( @@ -107,10 +107,10 @@ def create_batched_mm_kwargs( } # video metadata will be added back to the resized video data here. - text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data) + token_prompt = get_token_prompt(processor, resized_mm_data) mm_kwargs = processor( - prompt=token_prompt if text_prompt is None else text_prompt, + prompt=token_prompt, mm_items=processor.info.parse_mm_data(resized_mm_data), hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs, )["mm_kwargs"].require_data() diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index d8099c311a..c5583b669b 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -382,7 +382,6 @@ class AudioFlamingo3MultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, Any], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processor_mm_data = dict(mm_data) audios = processor_mm_data.pop("audios", None) @@ -393,7 +392,6 @@ class AudioFlamingo3MultiModalProcessor( prompt=prompt, mm_data=processor_mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if "input_features_mask" in outputs: diff --git a/vllm/model_executor/models/bagel.py b/vllm/model_executor/models/bagel.py index 28c8b64cd1..9603dc043c 100644 --- a/vllm/model_executor/models/bagel.py +++ b/vllm/model_executor/models/bagel.py @@ -273,15 +273,6 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]): class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]): """Multimodal processor for BAGEL model.""" - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index baa1a16a59..2f11b1dee0 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -472,7 +472,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if not mm_data: # HF processor always adds placeholders even when there's no image @@ -484,7 +483,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]): prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/chameleon.py b/vllm/model_executor/models/chameleon.py index 517697669c..af147b7f1a 100644 --- a/vllm/model_executor/models/chameleon.py +++ b/vllm/model_executor/models/chameleon.py @@ -140,7 +140,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if not mm_data: prompt_ids = self.info.get_tokenizer().encode(prompt) @@ -151,7 +150,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) def _apply_hf_processor_tokens_only( diff --git a/vllm/model_executor/models/cheers.py b/vllm/model_executor/models/cheers.py index 5f74c6771e..b9e88e1055 100644 --- a/vllm/model_executor/models/cheers.py +++ b/vllm/model_executor/models/cheers.py @@ -497,18 +497,8 @@ class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) - - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 2e01903505..4294a203df 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -208,40 +208,17 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]): timing_ctx: TimingContext, ) -> MultiModalInput: if inputs.mm_data_items: - if isinstance(inputs.prompt, str): - if len(inputs.prompt) > 0: - raise ValueError( - "CLIP accepts text-only or image-only inputs, not both! " - "You must pass an image with an empty text prompt." - ) + special_tokens = self.info.get_tokenizer().all_special_ids + if all(tok in special_tokens for tok in inputs.prompt): + inputs.prompt = [] else: - special_tokens = self.info.get_tokenizer().all_special_ids - if all(tok in special_tokens for tok in inputs.prompt): - inputs.prompt = [] - else: - raise ValueError( - "CLIP accepts text-only or image-only inputs, not both! " - "You must pass an image with an empty token prompt." - ) - - # For multi-modal data, the prompt after processing should - # only contain the dummy image tokens - inputs.tokenization_kwargs = { - **inputs.tokenization_kwargs, - "add_special_tokens": False, - } + raise ValueError( + "CLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty token prompt." + ) return super().apply(inputs, timing_ctx) - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index b6def10aab..147d462b23 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -227,13 +227,11 @@ class Cohere2VisionMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt, mm_data, mm_kwargs, - tok_kwargs, ) # Ensure num_patches is available for proper tensor splitting diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 7ffdd01cd6..9cbf9bb205 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -1936,9 +1936,9 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing def create_encoder_prompt( self, - prompt: str | list[int], + prompt: list[int], mm_items: MultiModalDataItems, - ) -> str | list[int]: + ) -> list[int]: return [0] def _call_hf_processor( @@ -1946,7 +1946,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ): if mm_data: feature_extractor = self.info.get_feature_extractor(**mm_kwargs) @@ -1959,7 +1958,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if "labels" in processed_outputs: processed_outputs["input_ids"] = processed_outputs.pop("labels") diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py index 798b87b77b..ffa10452c5 100644 --- a/vllm/model_executor/models/colmodernvbert.py +++ b/vllm/model_executor/models/colmodernvbert.py @@ -159,14 +159,12 @@ class ColModernVBertMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: tokenizer = self.info.get_tokenizer() assert isinstance(tokenizer, HfTokenizer) text_encoding = tokenizer( prompt, return_tensors="pt", - **tok_kwargs, ) result = BatchFeature(data=dict(text_encoding)) @@ -187,15 +185,6 @@ class ColModernVBertMultiModalProcessor( return result - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/colpali.py b/vllm/model_executor/models/colpali.py index 7948da01ba..aaf3b307a1 100644 --- a/vllm/model_executor/models/colpali.py +++ b/vllm/model_executor/models/colpali.py @@ -64,7 +64,6 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if mm_data: # The ColPali tokenizer_config.json ships with a small default @@ -72,13 +71,12 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor): # by PaliGemmaProcessor, causing a token-count mismatch. # vLLM enforces its own max_model_len, so we disable HF # truncation to keep all image + text tokens intact. - tok_kwargs = dict(tok_kwargs, truncation=False) - return super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, - ) + return self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**mm_kwargs), + dict(text=prompt, **mm_data), + dict(**mm_kwargs, truncation=False), + ) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL") diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index 4bb5598db3..5365b031f9 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -292,7 +292,6 @@ class DeepseekOCRMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if mm_data: processed_outputs = self.info.ctx.call_hf_processor( diff --git a/vllm/model_executor/models/deepseek_ocr2.py b/vllm/model_executor/models/deepseek_ocr2.py index aa6e9b1aec..d19af4713b 100644 --- a/vllm/model_executor/models/deepseek_ocr2.py +++ b/vllm/model_executor/models/deepseek_ocr2.py @@ -163,7 +163,6 @@ class DeepseekOCR2MultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if mm_data: processed_outputs = self.info.ctx.call_hf_processor( diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index 9ca05b6c77..ac83b6028a 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -241,7 +241,6 @@ class DeepseekVL2MultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if not mm_data: tokenizer = self.info.get_tokenizer() @@ -252,7 +251,6 @@ class DeepseekVL2MultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) processed_outputs["num_patches"] = ( diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index 8c4b150e30..4b495fae7a 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -1115,7 +1115,6 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # when the prompt is not empty but the multimodal data is empty, # directly invoke the tokenizer. @@ -1151,7 +1150,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin processor_output = self.info.ctx.call_hf_processor( hf_processor, dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]), - dict(**mm_kwargs, **tok_kwargs), + mm_kwargs, ) # Divide the processor_output into two modalities: image and video. diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index 7dedda3c3c..670e44cb86 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -233,7 +233,6 @@ class FireRedASR2MultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if mm_data: feature_extractor = self.info.get_feature_extractor(**mm_kwargs) @@ -246,7 +245,6 @@ class FireRedASR2MultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if "labels" in processed_outputs: processed_outputs["input_ids"] = processed_outputs.pop("labels") diff --git a/vllm/model_executor/models/fireredlid.py b/vllm/model_executor/models/fireredlid.py index 9f1786327b..32efe3dc0e 100644 --- a/vllm/model_executor/models/fireredlid.py +++ b/vllm/model_executor/models/fireredlid.py @@ -445,9 +445,9 @@ class FireRedLIDMultiModalProcessor( ): def create_encoder_prompt( self, - prompt: str | list[int], + prompt: list[int], mm_items: MultiModalDataItems, - ) -> str | list[int]: + ) -> list[int]: # Dummy encoder prompt for profiling (encoder only processes audio). return [0] @@ -456,7 +456,6 @@ class FireRedLIDMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if mm_data: feature_extractor = self.info.get_feature_extractor(**mm_kwargs) @@ -469,7 +468,6 @@ class FireRedLIDMultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if "labels" in processed_outputs: processed_outputs["input_ids"] = processed_outputs.pop("labels") diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 69f63d770b..205cb5c0e6 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -752,7 +752,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: if mm_data: feature_extractor = self.info.get_feature_extractor(**mm_kwargs) @@ -765,7 +764,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]): prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if "labels" in processed_outputs: processed_outputs["input_ids"] = processed_outputs.pop("labels") diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 10563f7efc..255e691df3 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -617,10 +617,9 @@ class FunAudioChatMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: tokenizer = self.info.get_tokenizer() - input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)]) + input_ids = torch.tensor([tokenizer.encode(prompt)]) audios = mm_data.get("audios", []) if not audios: @@ -679,15 +678,6 @@ class FunAudioChatMultiModalProcessor( return BatchFeature({"input_ids": input_ids, **mm_inputs}) - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 5551e0d8b9..31d3e82575 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -266,13 +266,11 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt, mm_data, mm_kwargs, - tok_kwargs, ) # HF processor pops the `num_crops` kwarg, which is needed by vLLM diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index 3a01f1457a..787642fa1e 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -262,7 +262,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]) prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # HF Transformers audio processor no longer accepts `audios` key. # We pop `audios` and replace it with `audio` key to suppress @@ -273,7 +272,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]) prompt, mm_data, mm_kwargs, - tok_kwargs, ) if "input_features" in processed_outputs: diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index aa9669aeb1..e993b1922f 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -560,31 +560,11 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): - def _apply_hf_processor_text_only( - self, - prompt_text: str, - tokenization_kwargs: Mapping[str, object], - ) -> list[int]: - # Bypass the HF processor and tokenize directly. The HF - # processor expands multimodal placeholders (<|video|>, etc.) - # via get_text_with_replacements, which raises StopIteration - # when the prompt contains placeholders without matching data. - # The text-only path only needs token IDs, so the tokenizer - # alone is sufficient. - processor = self.info.get_hf_processor() - text_inputs = processor.tokenizer([prompt_text], **tokenization_kwargs) - input_ids = text_inputs["input_ids"] - if not isinstance(input_ids, list): - input_ids = input_ids.tolist() - (prompt_ids,) = input_ids - return prompt_ids - def _call_hf_processor( self, prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs) @@ -640,7 +620,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): prompt=dummy_prompt, mm_data={"images": frames}, mm_kwargs=video_mm_kwargs, - tok_kwargs=tok_kwargs, ) # Remap HF key name @@ -741,7 +720,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): prompt, mm_data, patched_mm_kwargs, - tok_kwargs, ) # HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'. diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 679c90728b..c9ab921a8f 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1581,7 +1581,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) if not mm_data: @@ -1602,7 +1601,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): prompt=prompt, mm_data=prepared_data, mm_kwargs=prepared_kwargs, - tok_kwargs=tok_kwargs, ) if ( @@ -1631,7 +1629,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): prompt="<|begin_of_video|><|video|><|end_of_video|>", mm_data=video_mm_data, mm_kwargs=video_mm_kwargs, - tok_kwargs=tok_kwargs, ) input_ids = video_outputs.pop("input_ids") if swap_video_frame_tokens: @@ -1659,7 +1656,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if swap_video_frame_tokens: input_ids = processed_outputs["input_ids"] diff --git a/vllm/model_executor/models/glm4v.py b/vllm/model_executor/models/glm4v.py index 2e3a301579..a8512f49fa 100644 --- a/vllm/model_executor/models/glm4v.py +++ b/vllm/model_executor/models/glm4v.py @@ -466,15 +466,6 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]): class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]): - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index df1b4e0d4b..d90bbb5634 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -755,7 +755,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) prompt: str, mm_data: dict[str, object], mm_kwargs: Mapping[str, Any], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Normalize input: handle deprecated key and list conversion. if "audios" in mm_data: @@ -782,7 +781,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) # Postprocess: rename mask and add chunk counts diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 0e10121274..626750bd96 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -185,7 +185,6 @@ class GraniteSpeechMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) audios = mm_data.pop("audios", []) @@ -198,7 +197,6 @@ class GraniteSpeechMultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) if "audio" in mm_data: diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index 252556e86c..4454ce47c1 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -736,7 +736,6 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: hf_processor = self.info.get_hf_processor(**mm_kwargs) # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens. @@ -751,7 +750,7 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn return self.info.ctx.call_hf_processor( hf_processor, dict(text=prompt, **mm_data), - dict(**mm_kwargs, **tok_kwargs), + mm_kwargs, ) def _get_prompt_updates( diff --git a/vllm/model_executor/models/hyperclovax_vision.py b/vllm/model_executor/models/hyperclovax_vision.py index 647103aa44..e469c02022 100644 --- a/vllm/model_executor/models/hyperclovax_vision.py +++ b/vllm/model_executor/models/hyperclovax_vision.py @@ -195,7 +195,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: for video_idx, video_arr in enumerate(mm_data.get("videos", [])): if video_arr.dtype != np.uint8: @@ -263,15 +262,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn return processed_outputs - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/hyperclovax_vision_v2.py b/vllm/model_executor/models/hyperclovax_vision_v2.py index 6cfeac67a5..e50faf8855 100644 --- a/vllm/model_executor/models/hyperclovax_vision_v2.py +++ b/vllm/model_executor/models/hyperclovax_vision_v2.py @@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing ) dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False) + tokenizer = self.info.get_tokenizer() + prompt = tokenizer.encode( + prompt_text, + **self.info.default_tok_params.get_encode_kwargs(), + ) + return ProcessorInputs( - prompt=prompt_text, + prompt=prompt, mm_data_items=dummy_mm_items, hf_processor_mm_kwargs=mm_processor_kwargs or {}, - tokenization_kwargs={"truncation": False}, ) def get_dummy_mm_data( @@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: images = mm_data.get("images") videos = mm_data.get("videos") @@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor( # Build data dict for HF processor (images/videos only) # NOTE: We pass the prompt as-is without token normalization. - # Token expansion is handled by vLLM via _get_prompt_updates since - # _hf_processor_applies_updates returns False. + # Token expansion is handled by vLLM via _get_prompt_updates. data: dict[str, object] = dict( text=prompt, images=images, @@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor( processed_outputs = self.info.ctx.call_hf_processor( hf_processor=hf_processor, data=data, - kwargs=dict(**mm_kwargs, **tok_kwargs), + kwargs=mm_kwargs, ) return processed_outputs - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - # Match BaseMultiModalProcessor behavior: - # - raw multimodal inputs: HF processor applies updates - # - embedding inputs: vLLM applies updates - return super()._hf_processor_applies_updates( - prompt_text, - mm_items, - hf_processor_mm_kwargs, - tokenization_kwargs, - ) - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 7b3e552f0c..ab6c64f46f 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -244,7 +244,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Text-only input not supported in composite processor if not (images := mm_data.get("images", [])): @@ -260,7 +259,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo prompt, mm_data, mm_kwargs, - tok_kwargs, ) mm_items = self.info.parse_mm_data({"image": images}, validate=False) diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 6dbc358e08..bb3ba6a556 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -337,7 +337,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) videos = mm_data.pop("videos", []) @@ -364,7 +363,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo prompt=hf_processor.image_token, mm_data={"images": image}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) image_pixel_values.append(processed_outputs.pop("pixel_values")) @@ -387,7 +385,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo prompt=hf_processor.video_token, mm_data={"videos": video}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) video_pixel_values.append(processed_outputs.pop("pixel_values")) @@ -406,7 +403,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo prompt = re.sub("", hf_processor.image_token, prompt) prompt = re.sub("", hf_processor.video_token, prompt) - text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt") + text_outputs = tokenizer(prompt, return_tensors="pt") return BatchFeature({**text_outputs, **image_outputs, **video_outputs}) diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index d360e50182..f818ce9dfd 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -225,13 +225,11 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) hf_processor = self.info.get_hf_processor(**mm_kwargs) @@ -462,11 +460,8 @@ class InternVLMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt, mm_data, mm_kwargs, tok_kwargs - ) + processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs) hf_processor = self.info.get_hf_processor(**mm_kwargs) if (video_token_id := hf_processor.ctx_video_token_id) is not None: diff --git a/vllm/model_executor/models/jina_vl.py b/vllm/model_executor/models/jina_vl.py index 6970f74a27..c1cdf207e3 100644 --- a/vllm/model_executor/models/jina_vl.py +++ b/vllm/model_executor/models/jina_vl.py @@ -59,7 +59,6 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # NOTE: We should reverse the order of the mm_data because the # query prompt is placed after the document prompt in the score @@ -67,7 +66,7 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor): # stored in the opposite order (query first, then document). for _, value in mm_data.items(): value.reverse() - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) @MULTIMODAL_REGISTRY.register_processor( diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index 7a1421b515..8899914997 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -466,7 +466,6 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: """Run the underlying HF processor on text and image data.""" # Text-only input is handled as a special case here. diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index d64d2ebc64..dd894e60b1 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -1167,11 +1167,10 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Override to use the text path instead of token path to use the # video-specific logic in processing_keye.py - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index a4dcd729db..80a94373cb 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -383,11 +383,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Override to use the text path instead of token path to use the # video-specific logic in processing_keye.py - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index c999a694ad..60534658ae 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -231,7 +231,6 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: """Call the HuggingFace processor.""" # Convert mm_data format: {'audios': [...]} -> {'audio': ...} @@ -256,7 +255,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn return self.info.ctx.call_hf_processor( self.info.get_hf_processor(**mm_kwargs), dict(text=prompt, **mm_data), - dict(**mm_kwargs, **tok_kwargs), + mm_kwargs, ) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index b5c1c925d1..3054fcc5d8 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -263,11 +263,10 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo]) prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Override to use the text path instead of token path because vision chunk # is not considered - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index d247fef4d7..85286712fe 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -398,7 +398,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Text-only input not supported in composite processor if not (images := mm_data.get("images", [])): @@ -412,7 +411,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]): prompt, mm_data, mm_kwargs, - tok_kwargs, ) mm_items = self.info.parse_mm_data({"image": images}, validate=False) diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index c1ee640f63..83133e400f 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -44,13 +44,11 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) # NOTE: LightOnOCR does not use break/end tokens, so we remove them here. diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 1e850a7efc..9ef36e1824 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -326,13 +326,11 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) pixel_values = processed_outputs.get("pixel_values") diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 1beec4207d..7b8896f9e0 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -326,7 +326,6 @@ class LlavaOnevisionMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) videos = mm_data.pop("videos", []) @@ -337,7 +336,6 @@ class LlavaOnevisionMultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) # LLaVA-OneVision processor doesn't support multiple videos @@ -352,7 +350,6 @@ class LlavaOnevisionMultiModalProcessor( prompt=prompt, mm_data={}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) images = mm_data.pop("images", []) @@ -362,7 +359,6 @@ class LlavaOnevisionMultiModalProcessor( prompt=image_token * len(images), mm_data={"images": images}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) image_outputs = { k: v @@ -378,7 +374,6 @@ class LlavaOnevisionMultiModalProcessor( prompt=video_token, mm_data={"videos": video}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) pixel_values_videos.append(item_outputs["pixel_values_videos"][0]) @@ -392,22 +387,6 @@ class LlavaOnevisionMultiModalProcessor( ) return BatchFeature(combined_outputs) - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - base_result = super()._hf_processor_applies_updates( - prompt_text=prompt_text, - mm_items=mm_items, - hf_processor_mm_kwargs=hf_processor_mm_kwargs, - tokenization_kwargs=tokenization_kwargs, - ) - - return base_result and mm_items.get_count("video", strict=False) == 0 - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index f094a168fb..be51b1f0f5 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -1552,16 +1552,13 @@ class LlavaOnevision2MultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # The wrapped OV2 processor is a bare custom class without the standard # ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path # fails; overriding this method routes the base class to call us # directly. hf_processor = self.info.get_hf_processor(**mm_kwargs) - merged_kwargs = self.info.ctx.get_merged_mm_kwargs( - dict(**mm_kwargs, **tok_kwargs) - ) + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) merged_kwargs.setdefault("return_tensors", "pt") call_kwargs = { k: v @@ -1638,7 +1635,6 @@ class LlavaOnevision2MultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs={**mm_kwargs, "video_backend": "codec"}, - tok_kwargs=tok_kwargs, ) data = dict(output) return BatchFeature( @@ -1736,7 +1732,6 @@ class LlavaOnevision2MultiModalProcessor( prompt=new_prompt, mm_data=merged_mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) data = dict(output) @@ -1803,7 +1798,6 @@ class LlavaOnevision2MultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) def _rename_codec_outputs_to_video( diff --git a/vllm/model_executor/models/midashenglm.py b/vllm/model_executor/models/midashenglm.py index 5ecc92e4d0..8ddfd05f72 100644 --- a/vllm/model_executor/models/midashenglm.py +++ b/vllm/model_executor/models/midashenglm.py @@ -588,7 +588,6 @@ class MiDashengLMMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, Any], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: audios = mm_data.pop("audios", []) @@ -622,7 +621,6 @@ class MiDashengLMMultiModalProcessor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index 28205f9da2..8f78409581 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -917,7 +917,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo. Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses @@ -1000,7 +999,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing mm_data = {**mm_data, "videos": converted} - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) def _get_prompt_updates( self, diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index dd6f8d2f56..3dd3f8fd3b 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -436,7 +436,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: if (audios := mm_data.get("audios")) is None: return {} @@ -453,7 +452,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing prompts=[self.info.audio_pattern] * len(parsed_audios), mm_data={"audios": [[audio] for audio in parsed_audios]}, mm_kwargs={**mm_kwargs, "chunk_input": True}, - tok_kwargs=tok_kwargs, out_keys={"audio_features", "audio_feature_lens"}, ) @@ -486,11 +484,10 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: return { - **super().process_mm_inputs(mm_data, mm_kwargs, tok_kwargs), - **self.process_audios(mm_data, mm_kwargs, tok_kwargs), + **super().process_mm_inputs(mm_data, mm_kwargs), + **self.process_audios(mm_data, mm_kwargs), } def _get_prompt_updates( diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index d7ab703f38..968490d4e9 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -869,7 +869,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: if (images := mm_data.get("images")) is None: return {} @@ -886,7 +885,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): prompts=[self.info.image_pattern] * len(parsed_images), mm_data={"images": [[image] for image in parsed_images]}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, ) @@ -896,7 +894,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: if (videos := mm_data.get("videos")) is None: return {} @@ -918,7 +915,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): **mm_kwargs, "max_slice_nums": self.info.get_video_max_slice_num(), }, - tok_kwargs=tok_kwargs, out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, ) @@ -930,11 +926,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: return { - **self.process_images(mm_data, mm_kwargs, tok_kwargs), - **self.process_videos(mm_data, mm_kwargs, tok_kwargs), + **self.process_images(mm_data, mm_kwargs), + **self.process_videos(mm_data, mm_kwargs), } def _apply_prompt_updates( @@ -993,7 +988,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): prompts: list[str], mm_data: Mapping[str, Sequence[object]], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], *, out_keys: set[str], ) -> dict[str, NestedTensors]: @@ -1003,7 +997,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): prompt=prompts, # type: ignore mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) else: inputs = defaultdict[str, list[torch.Tensor]](list) @@ -1013,7 +1006,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): prompt=prompt, mm_data={k: v[i] for k, v in mm_data.items()}, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) for k, v in inputs_one.items(): @@ -1027,12 +1019,11 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: tokenizer = self.info.get_tokenizer() - input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)]) - mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs) + input_ids = torch.tensor([tokenizer.encode(prompt)]) + mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs) return BatchFeature( { @@ -1041,15 +1032,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): } ) - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return False - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py index d0948715a9..501be46050 100644 --- a/vllm/model_executor/models/minicpmv4_6.py +++ b/vllm/model_executor/models/minicpmv4_6.py @@ -154,7 +154,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: if (images := mm_data.get("images")) is None: return {} @@ -221,7 +220,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): self, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> Mapping[str, NestedTensors]: if (videos := mm_data.get("videos")) is None: return {} diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 025ce56408..22adf1c484 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -238,13 +238,11 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) pixel_values = processed_outputs.get("pixel_values") diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index fe5ff17a3b..04748465e0 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -594,13 +594,11 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]) prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processed_outputs = super()._call_hf_processor( prompt=prompt, mm_data=mm_data, mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, ) processor = self.info.get_hf_processor(**mm_kwargs) diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 3fccb7a86c..2b963e339a 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -1128,13 +1128,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: hf_processor = self.info.get_hf_processor(**mm_kwargs) processed_outputs = self.info.ctx.call_hf_processor( hf_processor.process, dict(text=prompt, **mm_data), - dict(**mm_kwargs, **tok_kwargs), + mm_kwargs, ) tokenizer = hf_processor.tokenizer diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 42f5ebfdc5..861e31928b 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -1948,7 +1948,6 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) @@ -2008,7 +2007,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): video_outputs = self.info.ctx.call_hf_processor( patched_call, dict(text=VIDEO_PROMPT, **video_mm_data), - dict(**video_mm_kwargs, **tok_kwargs), + video_mm_kwargs, ) input_ids = video_outputs.pop("input_ids") @@ -2058,7 +2057,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]): processed_outputs = self.info.ctx.call_hf_processor( patched_call, dict(text=prompt, **mm_data), - dict(**mm_kwargs, **tok_kwargs), + mm_kwargs, ) if (images := mm_data.get("images")) is not None: diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index d5f3e6b195..0464150bd4 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -947,11 +947,10 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: # Moondream3's processor handles images directly rather than exposing a # separate `image_processor`, so keep the cache path on text+MM calls. - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) @cached_property def bos_image_placeholder_tokens(self) -> list[int]: @@ -977,18 +976,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing "tilings": MultiModalFieldConfig.batched("image", keep_on_cpu=True), } - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - # Moondream3 HF processor does NOT expand placeholder tokens. - # vLLM expands BOS + so the whole HF image prefix is marked - # bidirectional by the multimodal prefix-LM mask. - return False - def _get_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 324b3f55ae..ceed6245a1 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -49,6 +49,7 @@ from vllm.multimodal.processing import ( BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, + InputProcessingContext, PromptReplacement, PromptUpdate, PromptUpdateDetails, @@ -102,6 +103,34 @@ MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = { "enable_time_marker", "mel_config", } +MOSS_AUDIO_PLACEHOLDER_TOKENS = ( + MOSS_AUDIO_BOS_TOKEN, + MOSS_AUDIO_TOKEN, + MOSS_AUDIO_EOS_TOKEN, +) + + +def _ensure_moss_audio_placeholder_tokens(tokenizer: object) -> None: + """Register the audio placeholder tokens if the tokenizer lacks them. + + The MOSS-Audio hub tokenizer does not define the audio placeholder + tokens (the reference HF processor patches them in at runtime). Left + as plain text, they merge with adjacent tokens under BPE, so prompts + no longer contain a stable token sequence for prompt updates to match. + """ + convert_tokens_to_ids = getattr(tokenizer, "convert_tokens_to_ids", None) + add_tokens = getattr(tokenizer, "add_tokens", None) + if convert_tokens_to_ids is None or add_tokens is None: + return + + unk_token_id = getattr(tokenizer, "unk_token_id", None) + missing_tokens = [ + token + for token in MOSS_AUDIO_PLACEHOLDER_TOKENS + if convert_tokens_to_ids(token) in (None, unk_token_id) + ] + if missing_tokens: + add_tokens(missing_tokens, special_tokens=True) class MossAudioAudioInputs(TensorSchema): @@ -1157,6 +1186,11 @@ class MossAudioProcessor: class MossAudioProcessingInfo(BaseProcessingInfo): + def __init__(self, ctx: InputProcessingContext) -> None: + super().__init__(ctx) + if ctx.tokenizer is not None: + _ensure_moss_audio_placeholder_tokens(ctx.tokenizer) + def get_hf_config(self) -> MossAudioConfig: config = self.ctx.get_hf_config() if isinstance(config, MossAudioConfig): @@ -1306,7 +1340,6 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) audios = mm_data.pop("audios", []) @@ -1314,15 +1347,10 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn mm_data["audio"] = audios mm_kwargs = dict(mm_kwargs) processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs) - tok_kwargs = { - key: value - for key, value in tok_kwargs.items() - if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS - } return self.info.ctx.call_hf_processor( self.info.get_hf_processor(**processor_kwargs), dict(text=prompt, **mm_data), - dict(**tok_kwargs), + {}, ) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index a862c27e6f..421901d0ee 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -466,33 +466,20 @@ class MossTranscribeDiarizeMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: tokenizer = self.info.get_tokenizer() audios = _get_audios_from_mm_data(mm_data) if not audios: - input_ids = tokenizer.encode( - prompt, - add_special_tokens=tok_kwargs.get("add_special_tokens", False), - ) + input_ids = tokenizer.encode(prompt, add_special_tokens=False) return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt") processed = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**mm_kwargs), dict(text=prompt, audio=audios), - dict(**mm_kwargs, **tok_kwargs), + mm_kwargs, ) return _add_vllm_audio_metadata(processed, len(audios)) - def _hf_processor_applies_updates( - self, - prompt_text: str, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - tokenization_kwargs: Mapping[str, object], - ) -> bool: - return mm_items.get_count("audio", strict=False) > 0 - def _get_mm_fields_config( self, hf_inputs: BatchFeature, diff --git a/vllm/model_executor/models/muse_glimmer.py b/vllm/model_executor/models/muse_glimmer.py index 83ec17a75d..fcdfb82333 100644 --- a/vllm/model_executor/models/muse_glimmer.py +++ b/vllm/model_executor/models/muse_glimmer.py @@ -262,23 +262,11 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing class MuseGlimmerMultiModalProcessor( BaseMultiModalProcessor[MuseGlimmerProcessingInfo] ): - def _apply_hf_processor_text_only( - self, - prompt_text: str, - tokenization_kwargs: Mapping[str, object], - ) -> list[int]: - tokenizer = self.info.get_tokenizer() - return tokenizer.encode( - prompt_text, - **{"add_special_tokens": False, **tokenization_kwargs}, - ) - def _call_hf_processor( self, prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: processor = self.info.get_hf_processor(**mm_kwargs) tokenizer = processor.tokenizer @@ -366,7 +354,6 @@ class MuseGlimmerMultiModalProcessor( video_pixel_values=video_pixels, video_feature_sizes=torch.tensor(video_sizes), ) - del tok_kwargs return BatchFeature(data=data, tensor_type=None) def _get_mm_fields_config( diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 7ecf8677bb..679495e3ec 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -366,14 +366,13 @@ class NanoNemotronVLMultiModalProcessor( prompt: str, mm_data: Mapping[str, object], mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], ) -> BatchFeature: """ Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`, so it chooses this path: `type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor` """ - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + return super()._call_hf_processor(prompt, mm_data, mm_kwargs) def _get_image_fields_config(self, hf_inputs: BatchFeature): if self.info.is_dynamic_tiler: @@ -714,10 +713,8 @@ class NanoNemotronVLMultiModalProcessor( if not audio_items: return super().apply(inputs, timing_ctx) - prompt = inputs.prompt tokenizer = self.info.get_tokenizer() - if not isinstance(prompt, str): - prompt = tokenizer.decode(prompt, skip_special_tokens=False) + prompt = tokenizer.decode(inputs.prompt, skip_special_tokens=False) # Inject AUDIO_CONTEXT only after