diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index d122427542..d64a4ef513 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -981,23 +981,50 @@ class Glm4vProcessingInfo(BaseProcessingInfo): def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor: return self.get_hf_processor(**kwargs).video_processor + def _get_processor_class_name(self) -> str | None: + from vllm.transformers_utils.processor import ( + get_processor_cls_name_from_config, + ) + from vllm.transformers_utils.utils import convert_model_repo_to_path + + return get_processor_cls_name_from_config( + convert_model_repo_to_path(self.ctx.model_config.model), + revision=self.ctx.model_config.revision, + ) + + @staticmethod + def _get_longest_edge(size: Any, config_name: str) -> int: + if isinstance(size, dict): + longest_edge = size.get("longest_edge") + else: + longest_edge = getattr(size, "longest_edge", None) + + if longest_edge is None: + raise ValueError(f"{config_name} must define longest_edge") + + return int(longest_edge) + def get_mm_max_tokens_per_item( self, seq_len: int, mm_counts: Mapping[str, int], ) -> Mapping[str, int] | None: - processor = self.get_hf_processor() - if isinstance(processor, Glm4vProcessor): + processor_class_name = self._get_processor_class_name() + if processor_class_name == "Glm4vProcessor": return None + if processor_class_name is None: + processor = self.get_hf_processor() + if isinstance(processor, Glm4vProcessor): + return None + result: dict[str, int] = {} if mm_counts.get("image", 0) > 0: result["image"] = self.get_max_image_tokens() if mm_counts.get("video", 0) > 0: - video_processor = self.get_video_processor() - max_pixels = video_processor.size["longest_edge"] + max_pixels = self._get_video_max_pixels() vision_config = self.get_hf_config().vision_config temporal_patch_size = vision_config.temporal_patch_size @@ -1076,7 +1103,40 @@ class Glm4vProcessingInfo(BaseProcessingInfo): ``smart_resize`` as the ``max_pixels`` argument, which constrains ``t_bar * h_bar * w_bar <= max_pixels``. """ - return self.get_image_processor().size["longest_edge"] + mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: + return int(override_max_pixels) + + image_processor_config = self.ctx.get_hf_image_processor_config() + if not image_processor_config.get("size"): + from transformers.image_processing_base import ImageProcessingMixin + + image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict( + self.ctx.model_config.model, + revision=self.ctx.model_config.revision, + ) + size = image_processor_config["size"] + if override_size := mm_kwargs.get("size"): + size = size | override_size + + return self._get_longest_edge(size, "GLM4V image processor size") + + def _get_video_max_pixels(self) -> int: + from transformers.video_processing_utils import BaseVideoProcessor + + mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: + return int(override_max_pixels) + + video_processor_config, _ = BaseVideoProcessor.get_video_processor_dict( + self.ctx.model_config.model, + revision=self.ctx.model_config.revision, + ) + size = video_processor_config["size"] + if override_size := mm_kwargs.get("size"): + size = size | override_size + + return self._get_longest_edge(size, "GLM4V video processor size") def get_image_size_with_most_features(self) -> ImageSize: # Use num_frames=1 for single-image budget estimation. @@ -1424,13 +1484,12 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]): num_videos = mm_counts.get("video", 0) hf_config = self.info.get_hf_config() - hf_processor = self.info.get_hf_processor() tokenizer = self.info.get_tokenizer() - image_token: str = hf_processor.image_token + image_token = tokenizer.decode([hf_config.image_token_id]) video_token_ids = [ hf_config.video_start_token_id, - hf_processor.video_token_id, + hf_config.video_token_id, hf_config.video_end_token_id, ] video_token = tokenizer.decode(video_token_ids)