diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index 0cafa261a3..8a77783282 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -93,7 +93,6 @@ def test_processor_override( limit_mm_per_prompt={"video": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - tokenizer = processor.info.get_tokenizer() hf_processor_mm_kwargs = {"fps": fps} # Build the image str / prompt based on the number of images we pass @@ -112,8 +111,8 @@ def test_processor_override( # Ensure we have the right number of placeholders per num_crops size hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - video_token_id = tokenizer.convert_tokens_to_ids(hf_processor.video_token) - video_tok_count = processed_inputs["prompt_token_ids"].count(video_token_id) + image_token_id = hf_processor.image_token_id + video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0] assert grid_t == expected_grid_t diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 810d9de87b..787e53e8df 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1403,6 +1403,11 @@ class Glm4vProcessingInfo(BaseProcessingInfo): timestamps_list = full_second_idxs[::2] return list(timestamps_list) + def _get_video_frame_embed_token_id(self, hf_processor: object) -> int: + if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA: + return hf_processor.image_token_id + return hf_processor.video_token_id + def _construct_video_placeholder( self, video_array: np.ndarray, @@ -1440,13 +1445,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo): num_tokens_per_frame = int(H * W) // merge_length placeholder = [] placeholder.append(bov_token_id) - # Glm46VProcessor uses image_token_id for video frame embeddings; - # Glm4vProcessor uses video_token_id. - frame_embed_token_id = ( - hf_processor.video_token_id - if isinstance(hf_processor, Glm4vProcessor) or not TRANSFORMERS_WITH_GA - else hf_processor.image_token_id - ) + frame_embed_token_id = self._get_video_frame_embed_token_id(hf_processor) for frame_idx in frames_idx_token: placeholder.append(boi_token_id) placeholder.extend([frame_embed_token_id] * num_tokens_per_frame) @@ -1604,11 +1603,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): processor = self.info.get_hf_processor(**mm_kwargs) - # Glm46VProcessor and GLMGA handle image/video placeholders together - # via the direct path. Only Glm4vProcessor (GLM-4.1V) needs the - # split-video path because it uses image_token_id as the video - # placeholder. The direct path requires transformers >= 5.5.0 - # (Glm46VProcessor / GlmgaVideoProcessor support). use_direct_path = ( not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA ) @@ -1630,6 +1624,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): ): video_grid_thw_lst = [] pixel_values_videos_lst = [] + frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor) + swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id for item in mm_data.pop("videos", []): video_array, metadata = item @@ -1650,9 +1646,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): tok_kwargs=tok_kwargs, ) input_ids = video_outputs.pop("input_ids") - input_ids[input_ids == processor.image_token_id] = ( - processor.video_token_id - ) + if swap_video_frame_tokens: + input_ids[input_ids == processor.image_token_id] = ( + processor.video_token_id + ) video_placeholder = processor.tokenizer.batch_decode(input_ids)[0] prompt = prompt.replace( "<|begin_of_video|><|video|><|end_of_video|>", @@ -1668,6 +1665,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): ) else: video_outputs = dict() + swap_video_frame_tokens = False processed_outputs = super()._call_hf_processor( prompt=prompt, @@ -1675,6 +1673,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): mm_kwargs=mm_kwargs, tok_kwargs=tok_kwargs, ) + if swap_video_frame_tokens: + input_ids = processed_outputs["input_ids"] + input_ids[input_ids == processor.video_token_id] = processor.image_token_id + processed_outputs["input_ids"] = input_ids combined_outputs = dict( processed_outputs, **video_outputs, @@ -1701,7 +1703,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): merge_length = image_processor.merge_size**2 - def get_image_replacement_glm4v(item_idx: int): + def get_image_replacement(item_idx: int): out_item = out_mm_kwargs["image"][item_idx] grid_thw = out_item["image_grid_thw"].data assert isinstance(grid_thw, torch.Tensor) @@ -1709,7 +1711,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): num_tokens = int(grid_thw.prod()) // merge_length return [hf_processor.image_token_id] * num_tokens - def get_video_replacement_glm4v(item_idx: int): + def get_video_replacement(item_idx: int): out_item = out_mm_kwargs["video"][item_idx] grid_thw = out_item["video_grid_thw"].data assert isinstance(grid_thw, torch.Tensor) @@ -1720,39 +1722,19 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): ) return PromptUpdateDetails.select_token_id( placeholder, - embed_token_id=hf_processor.video_token_id, + embed_token_id=self.info._get_video_frame_embed_token_id(hf_processor), ) - def get_video_replacement_glm46v(item_idx: int): - out_item = out_mm_kwargs["video"][item_idx] - grid_thw = out_item["video_grid_thw"].data - assert isinstance(grid_thw, torch.Tensor) - - video, metadata = mm_items["video"][item_idx] - placeholder = self.info._construct_video_placeholder( - video, metadata, grid_thw - ) - return PromptUpdateDetails.select_token_id( - placeholder, - embed_token_id=hf_processor.image_token_id, - ) - - is_glm46v = not isinstance(hf_processor, Glm4vProcessor) - return [ PromptReplacement( modality="image", target=hf_processor.image_token, - replacement=get_image_replacement_glm4v, + replacement=get_image_replacement, ), PromptReplacement( modality="video", target="<|begin_of_video|><|video|><|end_of_video|>", - replacement=( - get_video_replacement_glm46v - if is_glm46v and TRANSFORMERS_WITH_GA - else get_video_replacement_glm4v - ), + replacement=get_video_replacement, ), ]