Add Granite 4.1 Vision as built-in multimodal model (#40282)

Signed-off-by: Artem Spector <artems@il.ibm.com>
Signed-off-by: artemspector <artems@il.ibm.com>
Co-authored-by: artemspector <artems@il.ibm.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
artem-spector
2026-04-21 05:43:39 -07:00
committed by GitHub
co-authored by artemspector Claude Opus 4.6
parent d2e2e856ad
commit d249a9e90e
11 changed files with 1186 additions and 0 deletions
@@ -86,6 +86,29 @@ COMMON_BROADCAST_SETTINGS = {
# which cases would be selected and deselected by pytest. In general,
# this is a good idea for checking your command first, since tests are slow.
def _granite4_vision_vllm_to_hf_output(vllm_output, model):
"""Post-processor for granite4_vision vLLM output.
Self-contained to avoid calling AutoConfig/AutoTokenizer without
trust_remote_code (needed while the model is not in upstream HF).
"""
output_ids, output_str, out_logprobs = vllm_output
mm_token_id = 100352
hf_output_ids = [
token_id
for idx, token_id in enumerate(output_ids)
if token_id != mm_token_id or idx == 0 or output_ids[idx - 1] != mm_token_id
]
hf_output_str = (
output_str[1:] if output_str and output_str[0] == " " else output_str
)
eos_token_id = 100257
if hf_output_ids and hf_output_ids[-1] == eos_token_id:
hf_output_str = hf_output_str + "<|end_of_text|>"
return hf_output_ids, hf_output_str, out_logprobs
VLM_TEST_SETTINGS = {
#### Core tests to always run in the CI
"llava": VLMTestInfo(
@@ -492,6 +515,20 @@ VLM_TEST_SETTINGS = {
auto_cls=AutoModelForImageTextToText,
marks=[large_gpu_mark(min_gb=32)],
),
"granite4_vision": VLMTestInfo(
models=["ibm-granite/granite-vision-4.1-4b"],
test_type=(VLMTestType.IMAGE),
prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}\n<|assistant|>\n",
max_model_len=8192,
auto_cls=AutoModelForImageTextToText,
vllm_output_post_proc=_granite4_vision_vllm_to_hf_output,
image_size_factors=[(1.0,)],
vllm_runner_kwargs={
"enable_lora": True,
"max_lora_rank": 256,
"default_mm_loras": {"image": "ibm-granite/granite-vision-4.1-4b"},
},
),
"h2ovl": VLMTestInfo(
models=[
"h2oai/h2ovl-mississippi-800m",
+4
View File
@@ -899,6 +899,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"zai-org/GLM-ASR-Nano-2512",
min_transformers_version="5.0.0",
),
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-vision-4.1-4b",
is_available_online=False,
),
"GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"),
"GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-speech-3.3-2b",