Add Granite 4.1 Vision as built-in multimodal model (#40282)

Signed-off-by: Artem Spector <artems@il.ibm.com>
Signed-off-by: artemspector <artems@il.ibm.com>
Co-authored-by: artemspector <artems@il.ibm.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
artem-spector
2026-04-21 05:43:39 -07:00
committed by GitHub
co-authored by artemspector Claude Opus 4.6
parent d2e2e856ad
commit d249a9e90e
11 changed files with 1186 additions and 0 deletions
@@ -19,6 +19,7 @@ __all__ = [
"FireRedLIDProcessor",
"FunASRProcessor",
"GLM4VProcessor",
"Granite4VisionProcessor",
"H2OVLProcessor",
"HunYuanVLProcessor",
"HunYuanVLImageProcessor",
@@ -48,6 +49,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
"GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
"Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
"H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
"HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
"HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
from transformers import LlavaNextProcessor
from transformers.image_processing_utils import select_best_resolution
class Granite4VisionProcessor(LlavaNextProcessor):
"""Processor for Granite 4 Vision.
Extends LlavaNextProcessor to account for the Window Q-Former
downsampling when computing the number of image features.
This processor is needed because the granite4_vision processor type
is not yet in the transformers version pinned by vLLM.
"""
model_type = "granite4_vision"
def __init__(
self,
image_processor=None,
tokenizer=None,
patch_size=None,
vision_feature_select_strategy=None,
chat_template=None,
image_token="<image>",
num_additional_image_tokens=0,
downsample_rate=None,
**kwargs,
):
super().__init__(
image_processor=image_processor,
tokenizer=tokenizer,
patch_size=patch_size,
vision_feature_select_strategy=vision_feature_select_strategy,
chat_template=chat_template,
image_token=image_token,
num_additional_image_tokens=num_additional_image_tokens,
)
self.downsample_rate = downsample_rate
def _get_number_of_features(
self,
orig_height: int,
orig_width: int,
height: int,
width: int,
) -> int:
image_grid_pinpoints = self.image_processor.image_grid_pinpoints
height_best_resolution, width_best_resolution = select_best_resolution(
[orig_height, orig_width], image_grid_pinpoints
)
scale_height = height_best_resolution // height
scale_width = width_best_resolution // width
patches_height = height // self.patch_size
patches_width = width // self.patch_size
if self.downsample_rate is not None:
ds_rate = Fraction(self.downsample_rate)
patches_height = int(patches_height * ds_rate)
patches_width = int(patches_width * ds_rate)
unpadded_features, newline_features = self._get_unpadded_features(
orig_height,
orig_width,
patches_height,
patches_width,
scale_height,
scale_width,
)
base_features = (
patches_height * patches_width + self.num_additional_image_tokens
)
return unpadded_features + newline_features + base_features