From 7544286b04a860fcfb98725345f2b43a5023b844 Mon Sep 17 00:00:00 2001 From: Gonzague de Carpentier <82534773+decarpentierg@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:19:28 +0200 Subject: [PATCH] [Bugfix] Transformers backend: recompute `mm_token_type_ids` per request for M-RoPE (#46552) Signed-off-by: Gonzague de Carpentier Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../models/transformers/multimodal.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index d111af076d..0f80d56975 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -55,6 +55,8 @@ if TYPE_CHECKING: logger = init_logger(__name__) +_MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3} + class MultiModalProcessingInfo(BaseProcessingInfo): def get_supported_mm_limits(self): @@ -206,9 +208,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): ) # For gemma3 we check `token_type_ids` as the key - mm_token_type_ids = processed_data.get( - "mm_token_type_ids", processed_data.pop("token_type_ids", None) - ) + mm_token_type_ids = processed_data.pop("token_type_ids", None) + mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids) # We can infer vLLM style placeholder from token type ids, if we split # it for each input `mm_data`. @@ -377,7 +378,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): return None num_image_patches = kwargs.pop("num_image_patches") - kwargs.pop("mm_token_type_ids", None) # used only in `model.get_rope_index` if pixel_values is not None: # ROCm: Force math SDP backend for vision encoder to avoid accuracy issues @@ -468,24 +468,18 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): { "image_grid_thw", "video_grid_thw", - "mm_token_type_ids", "second_per_grid_ts", "audio_feature_lengths", "use_audio_in_video", }, ) - if any( - v - for k, v in kwargs.items() - if k not in {"image_grid_thw", "mm_token_type_ids"} - ): + if any(v for k, v in kwargs.items() if k not in {"image_grid_thw"}): raise NotImplementedError( "Transformers modeling backend only supports images." ) image_grid_thw = kwargs.get("image_grid_thw", []) video_grid_thw = kwargs.get("video_grid_thw", []) - mm_token_type_ids = kwargs.get("mm_token_type_ids") image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)( image_grid_thw @@ -494,8 +488,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): video_grid_thw ) - # In v4 `get_rope_index` doesn't have wildcard `kwargs`, and - # can't accept arbitrary args, even if its value is `None` + # `get_rope_index` doesn't always accept arbitrary `kwargs` kwargs = {} if not hasattr(self, "_get_rope_index_accepts_mm_token_type_ids"): import inspect @@ -507,11 +500,13 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) ) if self._get_rope_index_accepts_mm_token_type_ids: - if mm_token_type_ids: - kwargs["mm_token_type_ids"] = torch.cat(mm_token_type_ids) - else: - shape = (1, len(input_tokens)) - kwargs["mm_token_type_ids"] = torch.zeros(*shape, dtype=torch.int) + mm_token_type_ids = torch.zeros(len(input_tokens), dtype=torch.int) + for feature in mm_features: + position = feature.mm_position + offset, length = position.offset, position.length + mm_token_type_id = _MODALITY_TO_TOKEN_TYPE_ID[feature.modality] + mm_token_type_ids[offset : offset + length] = mm_token_type_id + kwargs["mm_token_type_ids"] = mm_token_type_ids.unsqueeze(0) mrope_positions, mrope_position_delta = self.model.get_rope_index( input_ids=torch.tensor(input_tokens).unsqueeze(0),