[2/3] Refactor InternVL-based processors (#37324)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -9,6 +9,7 @@
|
||||
# --------------------------------------------------------
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import cached_property
|
||||
from typing import Annotated, Literal, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
@@ -45,8 +46,9 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
BaseInternVLProcessor,
|
||||
InternVLImageProcessor,
|
||||
InternVLProcessor,
|
||||
InternVLVideoProcessor,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -123,7 +125,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
"""Basic image-only ProcessingInfo for InternVL-style models."""
|
||||
|
||||
@abstractmethod
|
||||
def get_hf_processor(self, **kwargs: object) -> BaseInternVLProcessor:
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -134,7 +136,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
processor: BaseInternVLProcessor,
|
||||
processor: InternVLProcessor,
|
||||
) -> int:
|
||||
return processor.get_num_image_tokens(
|
||||
image_width=image_width,
|
||||
@@ -143,8 +145,9 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
processor = self.get_hf_processor()
|
||||
image_processor = processor.image_processor
|
||||
|
||||
base_size = processor.image_size
|
||||
base_size = image_processor.image_size
|
||||
target_ratios = processor.resolve_target_ratios()
|
||||
|
||||
largest_feature_size, largest_feature_pinpoint = 0, None
|
||||
@@ -226,7 +229,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
image_token_id = hf_processor.ctx_image_token_id
|
||||
|
||||
# Since there may be extra tokens in the feature placeholders,
|
||||
# we need to pass the image token ID to the model to select the
|
||||
@@ -291,7 +294,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
return hf_processor.get_image_repl(feature_size, num_patches)
|
||||
return hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
@@ -305,23 +308,73 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
class InternVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""InternVL ProcessingInfo extended for video processing"""
|
||||
|
||||
@property
|
||||
def supports_video(self):
|
||||
return self.get_hf_processor().supports_video
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
def get_supported_mm_limits(self):
|
||||
video_limit = {"video": None} if self.supports_video else {}
|
||||
return {**super().get_supported_mm_limits(), **video_limit}
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
def get_video_token(self) -> str | None:
|
||||
return InternVLImageProcessor(**kwargs)
|
||||
|
||||
def get_video_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
|
||||
return InternVLVideoProcessor(**kwargs)
|
||||
|
||||
@cached_property
|
||||
def ctx_video_token(self):
|
||||
text_model_type = self.get_hf_config().get_text_config().model_type
|
||||
video_token_map = {
|
||||
ctx_video_token_map = {
|
||||
"qwen2": "<|video_pad|>",
|
||||
"qwen3": "<|video_pad|>",
|
||||
"qwen3_moe": "<|video_pad|>",
|
||||
"gpt_oss": "<|reserved_200000|>",
|
||||
}
|
||||
return video_token_map.get(text_model_type)
|
||||
|
||||
if text_model_type not in ctx_video_token_map:
|
||||
return None
|
||||
|
||||
ctx_video_token = ctx_video_token_map[text_model_type]
|
||||
if ctx_video_token not in self.get_tokenizer().get_vocab():
|
||||
return None
|
||||
|
||||
return ctx_video_token
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
ctx_video_token = self.ctx_video_token
|
||||
video_processor = (
|
||||
self.get_video_processor(**kwargs) if ctx_video_token else None
|
||||
)
|
||||
|
||||
return InternVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=image_processor,
|
||||
video_processor=video_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
ctx_video_token=ctx_video_token,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self):
|
||||
video_limit = {"video": None} if self.ctx_video_token else {}
|
||||
return {**super().get_supported_mm_limits(), **video_limit}
|
||||
|
||||
def get_num_frames_with_most_features(
|
||||
self,
|
||||
@@ -332,22 +385,14 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
max_videos = mm_counts.get("video", 0)
|
||||
|
||||
processor = self.get_hf_processor()
|
||||
num_image_token = processor.image_seq_length
|
||||
|
||||
max_image_tokens = self.get_max_image_tokens() * max_images
|
||||
max_total_frames = (seq_len - max_image_tokens) // processor.num_image_token
|
||||
max_total_frames = (seq_len - max_image_tokens) // num_image_token
|
||||
max_frames_per_video = max_total_frames // max(max_videos, 1)
|
||||
|
||||
return max(max_frames_per_video, 1)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
return self.ctx.init_processor(
|
||||
InternVLProcessor,
|
||||
config=self.get_hf_config(),
|
||||
tokenizer=self.get_tokenizer(),
|
||||
video_token=self.get_video_token(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class InternVLDummyInputsBuilder(
|
||||
BaseInternVLDummyInputsBuilder[InternVLProcessingInfo]
|
||||
@@ -366,7 +411,7 @@ class InternVLDummyInputsBuilder(
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
|
||||
if self.info.supports_video:
|
||||
if self.info.ctx_video_token:
|
||||
config = self.info.get_hf_config()
|
||||
image_size: int = config.vision_config.image_size
|
||||
target_num_frames = self.info.get_num_frames_with_most_features(
|
||||
@@ -405,11 +450,9 @@ class InternVLMultiModalProcessor(
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
if (
|
||||
self.info.supports_video
|
||||
and (video_token_id := hf_processor.video_token_id) is not None
|
||||
):
|
||||
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
|
||||
processed_outputs["video_token_id"] = torch.tensor(video_token_id)
|
||||
|
||||
return processed_outputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -418,7 +461,7 @@ class InternVLMultiModalProcessor(
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
image_fields = super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs)
|
||||
if self.info.supports_video:
|
||||
if self.info.ctx_video_token:
|
||||
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
|
||||
num_videos = len(video_num_patches)
|
||||
video_fields = dict(
|
||||
@@ -444,6 +487,8 @@ class InternVLMultiModalProcessor(
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
out_mm_kwargs=out_mm_kwargs,
|
||||
)
|
||||
if self.info.ctx_video_token is None:
|
||||
return prompt_repl
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
@@ -456,26 +501,20 @@ class InternVLMultiModalProcessor(
|
||||
video_num_patches = []
|
||||
|
||||
def get_video_replacement_internvl(item_idx: int):
|
||||
feature_size = hf_processor.num_image_token
|
||||
num_patches = video_num_patches[item_idx]
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
return hf_processor.get_video_repl(
|
||||
feature_size, num_patches, video_context_token=hf_processor.video_token
|
||||
)
|
||||
return hf_processor.get_video_repl(num_patches)
|
||||
|
||||
if self.info.supports_video:
|
||||
prompt_repl = [
|
||||
*prompt_repl,
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
replacement=get_video_replacement_internvl,
|
||||
),
|
||||
]
|
||||
|
||||
return prompt_repl
|
||||
return [
|
||||
*prompt_repl,
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
replacement=get_video_replacement_internvl,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
Reference in New Issue
Block a user