From 2ee51063722c9e6d28f71340966942394f330c0e Mon Sep 17 00:00:00 2001 From: Raushan Turganbay Date: Tue, 9 Jun 2026 17:01:04 +0200 Subject: [PATCH] Remove `raw_inputs` from transformers backend (#39425) Signed-off-by: raushan --- .../models/transformers/multimodal.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 338790dc69..3bcc1be7c8 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -234,12 +234,9 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): ) # For gemma3 we check `token_type_ids` as the key - token_type_key = ( - "mm_token_type_ids" - if "mm_token_type_ids" in processed_data - else "token_type_ids" + mm_token_type_ids = processed_data.get( + "mm_token_type_ids", processed_data.pop("token_type_ids", None) ) - mm_token_type_ids = processed_data.get(token_type_key) # We can infer vLLM style placeholder from token type ids, if we split # it for each input `mm_data`. @@ -292,8 +289,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): - supports_multimodal_raw_input_only = True - def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): # Skip SupportsMRoPE.__init__ and call the next class in MRO super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix) @@ -366,15 +361,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): inputs_embeds: torch.Tensor | None = None, **kwargs: object, ) -> torch.Tensor | IntermediateTensors: - # Gemma3 and PaliGemma needs `token_type_ids` to work correctly - # Other models will not have `token_type_ids` in kwargs - kwargs = {k: v for k, v in kwargs.items() if k == "token_type_ids"} # Positions shape handling for MRoPE models if self.model_config.uses_mrope: # [3, seq_len] -> [3, 1, seq_len] positions = positions[:, None] model_output = super().forward( - input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs + input_ids, positions, intermediate_tensors, inputs_embeds ) return model_output @@ -413,7 +405,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): return None num_image_patches = kwargs.pop("num_image_patches") - kwargs.pop("token_type_ids", None) # used only in `forward` kwargs.pop("mm_token_type_ids", None) # used only in `model.get_rope_index` if pixel_values is not None: