[2/3] Refactor InternVL-based processors (#37324)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -16,7 +16,10 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.model_executor.models.siglip import SiglipVisionModel
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.eagle2_5_vl import Eagle2_5_VLProcessor
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
InternVLImageProcessor,
|
||||
InternVLProcessor,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -68,12 +71,35 @@ Eagle2_5_VLImageInputs: TypeAlias = (
|
||||
class Eagle2_5_VLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""Processing info for Eagle2.5-VL model."""
|
||||
|
||||
def get_hf_processor(self, **kwargs) -> Eagle2_5_VLProcessor:
|
||||
return self.ctx.init_processor(
|
||||
Eagle2_5_VLProcessor,
|
||||
config=self.ctx.get_hf_config(),
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault(
|
||||
"image_size", config.force_image_size or vision_config.image_size
|
||||
)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
return InternVLImageProcessor(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs) -> InternVLProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return InternVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
**kwargs,
|
||||
image_processor=image_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -395,13 +395,13 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_size = vision_config["image_size"]
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("size", {"width": image_size, "height": image_size})
|
||||
|
||||
return GLM4VImageProcessorFast(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> GLM4VProcessor:
|
||||
return self.ctx.init_processor(
|
||||
GLM4VProcessor,
|
||||
return GLM4VProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=self.get_image_processor(**kwargs),
|
||||
)
|
||||
|
||||
@@ -28,7 +28,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.transformers_utils.processors.h2ovl import H2OVLProcessor
|
||||
from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
|
||||
|
||||
from .intern_vit import InternVisionModel
|
||||
from .internvl import (
|
||||
@@ -40,12 +40,34 @@ from .internvl import (
|
||||
|
||||
|
||||
class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
kwargs.setdefault("use_msac", config.use_msac)
|
||||
|
||||
return H2OVLImageProcessor(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> H2OVLProcessor:
|
||||
return self.ctx.init_processor(
|
||||
H2OVLProcessor,
|
||||
config=self.get_hf_config(),
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return H2OVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
**kwargs,
|
||||
image_processor=image_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
)
|
||||
|
||||
def get_num_image_tokens(
|
||||
@@ -106,7 +128,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
return hf_processor.get_image_repl(feature_size, num_patches)
|
||||
return hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
# --------------------------------------------------------
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import cached_property
|
||||
from typing import Annotated, Literal, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
@@ -45,8 +46,9 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
BaseInternVLProcessor,
|
||||
InternVLImageProcessor,
|
||||
InternVLProcessor,
|
||||
InternVLVideoProcessor,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -123,7 +125,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
"""Basic image-only ProcessingInfo for InternVL-style models."""
|
||||
|
||||
@abstractmethod
|
||||
def get_hf_processor(self, **kwargs: object) -> BaseInternVLProcessor:
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -134,7 +136,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
processor: BaseInternVLProcessor,
|
||||
processor: InternVLProcessor,
|
||||
) -> int:
|
||||
return processor.get_num_image_tokens(
|
||||
image_width=image_width,
|
||||
@@ -143,8 +145,9 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
processor = self.get_hf_processor()
|
||||
image_processor = processor.image_processor
|
||||
|
||||
base_size = processor.image_size
|
||||
base_size = image_processor.image_size
|
||||
target_ratios = processor.resolve_target_ratios()
|
||||
|
||||
largest_feature_size, largest_feature_pinpoint = 0, None
|
||||
@@ -226,7 +229,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
image_token_id = hf_processor.ctx_image_token_id
|
||||
|
||||
# Since there may be extra tokens in the feature placeholders,
|
||||
# we need to pass the image token ID to the model to select the
|
||||
@@ -291,7 +294,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
return hf_processor.get_image_repl(feature_size, num_patches)
|
||||
return hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
@@ -305,23 +308,73 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
class InternVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""InternVL ProcessingInfo extended for video processing"""
|
||||
|
||||
@property
|
||||
def supports_video(self):
|
||||
return self.get_hf_processor().supports_video
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
def get_supported_mm_limits(self):
|
||||
video_limit = {"video": None} if self.supports_video else {}
|
||||
return {**super().get_supported_mm_limits(), **video_limit}
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
def get_video_token(self) -> str | None:
|
||||
return InternVLImageProcessor(**kwargs)
|
||||
|
||||
def get_video_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
|
||||
return InternVLVideoProcessor(**kwargs)
|
||||
|
||||
@cached_property
|
||||
def ctx_video_token(self):
|
||||
text_model_type = self.get_hf_config().get_text_config().model_type
|
||||
video_token_map = {
|
||||
ctx_video_token_map = {
|
||||
"qwen2": "<|video_pad|>",
|
||||
"qwen3": "<|video_pad|>",
|
||||
"qwen3_moe": "<|video_pad|>",
|
||||
"gpt_oss": "<|reserved_200000|>",
|
||||
}
|
||||
return video_token_map.get(text_model_type)
|
||||
|
||||
if text_model_type not in ctx_video_token_map:
|
||||
return None
|
||||
|
||||
ctx_video_token = ctx_video_token_map[text_model_type]
|
||||
if ctx_video_token not in self.get_tokenizer().get_vocab():
|
||||
return None
|
||||
|
||||
return ctx_video_token
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
ctx_video_token = self.ctx_video_token
|
||||
video_processor = (
|
||||
self.get_video_processor(**kwargs) if ctx_video_token else None
|
||||
)
|
||||
|
||||
return InternVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=image_processor,
|
||||
video_processor=video_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
ctx_video_token=ctx_video_token,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self):
|
||||
video_limit = {"video": None} if self.ctx_video_token else {}
|
||||
return {**super().get_supported_mm_limits(), **video_limit}
|
||||
|
||||
def get_num_frames_with_most_features(
|
||||
self,
|
||||
@@ -332,22 +385,14 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
max_videos = mm_counts.get("video", 0)
|
||||
|
||||
processor = self.get_hf_processor()
|
||||
num_image_token = processor.image_seq_length
|
||||
|
||||
max_image_tokens = self.get_max_image_tokens() * max_images
|
||||
max_total_frames = (seq_len - max_image_tokens) // processor.num_image_token
|
||||
max_total_frames = (seq_len - max_image_tokens) // num_image_token
|
||||
max_frames_per_video = max_total_frames // max(max_videos, 1)
|
||||
|
||||
return max(max_frames_per_video, 1)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
return self.ctx.init_processor(
|
||||
InternVLProcessor,
|
||||
config=self.get_hf_config(),
|
||||
tokenizer=self.get_tokenizer(),
|
||||
video_token=self.get_video_token(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class InternVLDummyInputsBuilder(
|
||||
BaseInternVLDummyInputsBuilder[InternVLProcessingInfo]
|
||||
@@ -366,7 +411,7 @@ class InternVLDummyInputsBuilder(
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
|
||||
if self.info.supports_video:
|
||||
if self.info.ctx_video_token:
|
||||
config = self.info.get_hf_config()
|
||||
image_size: int = config.vision_config.image_size
|
||||
target_num_frames = self.info.get_num_frames_with_most_features(
|
||||
@@ -405,11 +450,9 @@ class InternVLMultiModalProcessor(
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
if (
|
||||
self.info.supports_video
|
||||
and (video_token_id := hf_processor.video_token_id) is not None
|
||||
):
|
||||
if (video_token_id := hf_processor.ctx_video_token_id) is not None:
|
||||
processed_outputs["video_token_id"] = torch.tensor(video_token_id)
|
||||
|
||||
return processed_outputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -418,7 +461,7 @@ class InternVLMultiModalProcessor(
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
image_fields = super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs)
|
||||
if self.info.supports_video:
|
||||
if self.info.ctx_video_token:
|
||||
video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
|
||||
num_videos = len(video_num_patches)
|
||||
video_fields = dict(
|
||||
@@ -444,6 +487,8 @@ class InternVLMultiModalProcessor(
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
out_mm_kwargs=out_mm_kwargs,
|
||||
)
|
||||
if self.info.ctx_video_token is None:
|
||||
return prompt_repl
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
@@ -456,26 +501,20 @@ class InternVLMultiModalProcessor(
|
||||
video_num_patches = []
|
||||
|
||||
def get_video_replacement_internvl(item_idx: int):
|
||||
feature_size = hf_processor.num_image_token
|
||||
num_patches = video_num_patches[item_idx]
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
return hf_processor.get_video_repl(
|
||||
feature_size, num_patches, video_context_token=hf_processor.video_token
|
||||
)
|
||||
return hf_processor.get_video_repl(num_patches)
|
||||
|
||||
if self.info.supports_video:
|
||||
prompt_repl = [
|
||||
*prompt_repl,
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
replacement=get_video_replacement_internvl,
|
||||
),
|
||||
]
|
||||
|
||||
return prompt_repl
|
||||
return [
|
||||
*prompt_repl,
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
replacement=get_video_replacement_internvl,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -26,8 +26,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_image_processor_from_config
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
LlamaNemotronNanoVLImageProcessor,
|
||||
LlamaNemotronNanoVLProcessor,
|
||||
LlamaNemotronVLEmbedImageProcessor,
|
||||
LlamaNemotronVLEmbedProcessor,
|
||||
NemotronVLProcessor,
|
||||
)
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
@@ -50,19 +52,34 @@ from .utils import (
|
||||
class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""Processing info for Nemotron VL models."""
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> NemotronVLProcessor:
|
||||
return self.ctx.init_processor(
|
||||
NemotronVLProcessor,
|
||||
config=self.get_hf_config(),
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=self.get_image_processor(),
|
||||
**kwargs,
|
||||
def get_image_processor(self, **kwargs: object):
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
orig_processor = cached_image_processor_from_config(
|
||||
self.ctx.model_config, **kwargs
|
||||
)
|
||||
|
||||
def get_image_processor(self, **kwargs: object):
|
||||
return cached_image_processor_from_config(
|
||||
self.ctx.model_config,
|
||||
**kwargs,
|
||||
return LlamaNemotronNanoVLImageProcessor(
|
||||
image_size=orig_processor.image_size,
|
||||
min_dynamic_patch=1,
|
||||
max_dynamic_patch=orig_processor.max_num_tiles,
|
||||
dynamic_image_size=True,
|
||||
use_thumbnail=orig_processor.use_thumbnail,
|
||||
)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> LlamaNemotronNanoVLProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return LlamaNemotronNanoVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=image_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
)
|
||||
|
||||
|
||||
@@ -386,29 +403,58 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
|
||||
# --------------------------------------------------------
|
||||
|
||||
|
||||
class LlamaNemotronVLEmbedProcessingInfo(NemotronVLProcessingInfo):
|
||||
class LlamaNemotronVLEmbedProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""Processing info for LlamaNemotronVL embedding model."""
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> LlamaNemotronVLEmbedProcessor:
|
||||
"""Override to create embedding-specific processor without image_processor."""
|
||||
def get_image_processor(self, **kwargs):
|
||||
model_config = self.ctx.model_config
|
||||
processor_config = {}
|
||||
if model_config.model is not None:
|
||||
processor_config = (
|
||||
get_hf_file_to_dict(
|
||||
"processor_config.json",
|
||||
model_config.model,
|
||||
model_config.revision,
|
||||
)
|
||||
or {}
|
||||
)
|
||||
|
||||
return self.ctx.init_processor(
|
||||
LlamaNemotronVLEmbedProcessor,
|
||||
config=self.get_hf_config(),
|
||||
config = self.get_hf_config()
|
||||
processor_config = (
|
||||
get_hf_file_to_dict(
|
||||
"processor_config.json",
|
||||
model_config.model,
|
||||
model_config.revision,
|
||||
)
|
||||
or {}
|
||||
)
|
||||
|
||||
min_dynamic_patch = processor_config.get(
|
||||
"min_input_tiles",
|
||||
getattr(config, "min_dynamic_patch", 1),
|
||||
)
|
||||
max_dynamic_patch = processor_config.get(
|
||||
"max_input_tiles",
|
||||
getattr(config, "max_dynamic_patch", 1),
|
||||
)
|
||||
dynamic_image_size = processor_config.get(
|
||||
"dynamic_image_size",
|
||||
getattr(config, "dynamic_image_size", True),
|
||||
)
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", config.force_image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", True)
|
||||
|
||||
return LlamaNemotronVLEmbedImageProcessor(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> LlamaNemotronVLEmbedProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return LlamaNemotronVLEmbedProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
processor_config=processor_config,
|
||||
**kwargs,
|
||||
image_processor=image_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -27,7 +27,8 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.transformers_utils.processors.nvlm_d import IMG_PAD, NVLMProcessor
|
||||
from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
|
||||
from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
|
||||
|
||||
from .intern_vit import InternVisionModel
|
||||
from .internvl import (
|
||||
@@ -39,12 +40,33 @@ from .internvl import (
|
||||
|
||||
|
||||
class NVLMProcessingInfo(BaseInternVLProcessingInfo):
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
return InternVLImageProcessor(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> NVLMProcessor:
|
||||
return self.ctx.init_processor(
|
||||
NVLMProcessor,
|
||||
config=self.get_hf_config(),
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return NVLMProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
**kwargs,
|
||||
image_processor=image_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
)
|
||||
|
||||
|
||||
@@ -117,9 +139,11 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
repl = hf_processor.get_image_repl(feature_size, num_patches)
|
||||
repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl.full + "\n", IMG_PAD)
|
||||
return PromptUpdateDetails.select_text(
|
||||
repl.full + "\n", hf_processor.ctx_image_token
|
||||
)
|
||||
|
||||
# See note in dummy data regarding why we have the extra newline
|
||||
return [
|
||||
|
||||
@@ -440,13 +440,13 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
|
||||
vision_config = config.visual
|
||||
|
||||
image_size = vision_config["image_size"]
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("size", {"width": image_size, "height": image_size})
|
||||
|
||||
return QwenVLImageProcessorFast(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
|
||||
return self.ctx.init_processor(
|
||||
QwenVLProcessor,
|
||||
return QwenVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=self.get_image_processor(**kwargs),
|
||||
)
|
||||
|
||||
@@ -43,7 +43,10 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.skyworkr1v import SkyworkR1VProcessor
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
InternVLImageProcessor,
|
||||
InternVLProcessor,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
@@ -96,12 +99,33 @@ SkyworkR1VImageInputs: TypeAlias = (
|
||||
|
||||
|
||||
class SkyworkR1VProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_processor(self, **kwargs: object) -> SkyworkR1VProcessor:
|
||||
return self.ctx.init_processor(
|
||||
SkyworkR1VProcessor,
|
||||
config=self.get_hf_config(),
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
return InternVLImageProcessor(**kwargs)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
image_processor = self.get_image_processor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return InternVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
**kwargs,
|
||||
image_processor=image_processor,
|
||||
image_seq_length=image_seq_length,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -112,7 +136,7 @@ class SkyworkR1VProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
processor: SkyworkR1VProcessor,
|
||||
processor: InternVLProcessor,
|
||||
) -> int:
|
||||
return processor.get_num_image_tokens(
|
||||
image_width=image_width,
|
||||
@@ -121,8 +145,9 @@ class SkyworkR1VProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
processor = self.get_hf_processor()
|
||||
image_processor = processor.image_processor
|
||||
|
||||
base_size = processor.image_size
|
||||
base_size = image_processor.image_size
|
||||
target_ratios = processor.resolve_target_ratios()
|
||||
|
||||
largest_feature_size, largest_feature_pinpoint = 0, None
|
||||
@@ -187,7 +212,7 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[SkyworkR1VProcessing
|
||||
)
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
image_token_id = hf_processor.ctx_image_token_id
|
||||
|
||||
# Since there may be extra tokens in the feature placeholders,
|
||||
# we need to pass the image token ID to the model to select the
|
||||
@@ -252,7 +277,7 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[SkyworkR1VProcessing
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
return hf_processor.get_image_repl(feature_size, num_patches)
|
||||
return hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
|
||||
Reference in New Issue
Block a user