[GLM4V] Avoid GLM4V processor init during startup metadata reads (#47155)
Signed-off-by: xiaoming <1259730330@qq.com>
This commit is contained in:
@@ -981,23 +981,50 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
||||
def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:
|
||||
return self.get_hf_processor(**kwargs).video_processor
|
||||
|
||||
def _get_processor_class_name(self) -> str | None:
|
||||
from vllm.transformers_utils.processor import (
|
||||
get_processor_cls_name_from_config,
|
||||
)
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
|
||||
return get_processor_cls_name_from_config(
|
||||
convert_model_repo_to_path(self.ctx.model_config.model),
|
||||
revision=self.ctx.model_config.revision,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_longest_edge(size: Any, config_name: str) -> int:
|
||||
if isinstance(size, dict):
|
||||
longest_edge = size.get("longest_edge")
|
||||
else:
|
||||
longest_edge = getattr(size, "longest_edge", None)
|
||||
|
||||
if longest_edge is None:
|
||||
raise ValueError(f"{config_name} must define longest_edge")
|
||||
|
||||
return int(longest_edge)
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int] | None:
|
||||
processor = self.get_hf_processor()
|
||||
if isinstance(processor, Glm4vProcessor):
|
||||
processor_class_name = self._get_processor_class_name()
|
||||
if processor_class_name == "Glm4vProcessor":
|
||||
return None
|
||||
|
||||
if processor_class_name is None:
|
||||
processor = self.get_hf_processor()
|
||||
if isinstance(processor, Glm4vProcessor):
|
||||
return None
|
||||
|
||||
result: dict[str, int] = {}
|
||||
|
||||
if mm_counts.get("image", 0) > 0:
|
||||
result["image"] = self.get_max_image_tokens()
|
||||
|
||||
if mm_counts.get("video", 0) > 0:
|
||||
video_processor = self.get_video_processor()
|
||||
max_pixels = video_processor.size["longest_edge"]
|
||||
max_pixels = self._get_video_max_pixels()
|
||||
|
||||
vision_config = self.get_hf_config().vision_config
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
@@ -1076,7 +1103,40 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
||||
``smart_resize`` as the ``max_pixels`` argument, which constrains
|
||||
``t_bar * h_bar * w_bar <= max_pixels``.
|
||||
"""
|
||||
return self.get_image_processor().size["longest_edge"]
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
return int(override_max_pixels)
|
||||
|
||||
image_processor_config = self.ctx.get_hf_image_processor_config()
|
||||
if not image_processor_config.get("size"):
|
||||
from transformers.image_processing_base import ImageProcessingMixin
|
||||
|
||||
image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
|
||||
self.ctx.model_config.model,
|
||||
revision=self.ctx.model_config.revision,
|
||||
)
|
||||
size = image_processor_config["size"]
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
|
||||
return self._get_longest_edge(size, "GLM4V image processor size")
|
||||
|
||||
def _get_video_max_pixels(self) -> int:
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
return int(override_max_pixels)
|
||||
|
||||
video_processor_config, _ = BaseVideoProcessor.get_video_processor_dict(
|
||||
self.ctx.model_config.model,
|
||||
revision=self.ctx.model_config.revision,
|
||||
)
|
||||
size = video_processor_config["size"]
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
|
||||
return self._get_longest_edge(size, "GLM4V video processor size")
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
# Use num_frames=1 for single-image budget estimation.
|
||||
@@ -1424,13 +1484,12 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
|
||||
hf_config = self.info.get_hf_config()
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
image_token: str = hf_processor.image_token
|
||||
image_token = tokenizer.decode([hf_config.image_token_id])
|
||||
video_token_ids = [
|
||||
hf_config.video_start_token_id,
|
||||
hf_processor.video_token_id,
|
||||
hf_config.video_token_id,
|
||||
hf_config.video_end_token_id,
|
||||
]
|
||||
video_token = tokenizer.decode(video_token_ids)
|
||||
|
||||
Reference in New Issue
Block a user