Add Granite 4.1 Vision as built-in multimodal model (#40282)
Signed-off-by: Artem Spector <artems@il.ibm.com> Signed-off-by: artemspector <artems@il.ibm.com> Co-authored-by: artemspector <artems@il.ibm.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
artemspector
Claude Opus 4.6
parent
d2e2e856ad
commit
d249a9e90e
@@ -86,6 +86,29 @@ COMMON_BROADCAST_SETTINGS = {
|
||||
# which cases would be selected and deselected by pytest. In general,
|
||||
# this is a good idea for checking your command first, since tests are slow.
|
||||
|
||||
|
||||
def _granite4_vision_vllm_to_hf_output(vllm_output, model):
|
||||
"""Post-processor for granite4_vision vLLM output.
|
||||
|
||||
Self-contained to avoid calling AutoConfig/AutoTokenizer without
|
||||
trust_remote_code (needed while the model is not in upstream HF).
|
||||
"""
|
||||
output_ids, output_str, out_logprobs = vllm_output
|
||||
mm_token_id = 100352
|
||||
hf_output_ids = [
|
||||
token_id
|
||||
for idx, token_id in enumerate(output_ids)
|
||||
if token_id != mm_token_id or idx == 0 or output_ids[idx - 1] != mm_token_id
|
||||
]
|
||||
hf_output_str = (
|
||||
output_str[1:] if output_str and output_str[0] == " " else output_str
|
||||
)
|
||||
eos_token_id = 100257
|
||||
if hf_output_ids and hf_output_ids[-1] == eos_token_id:
|
||||
hf_output_str = hf_output_str + "<|end_of_text|>"
|
||||
return hf_output_ids, hf_output_str, out_logprobs
|
||||
|
||||
|
||||
VLM_TEST_SETTINGS = {
|
||||
#### Core tests to always run in the CI
|
||||
"llava": VLMTestInfo(
|
||||
@@ -492,6 +515,20 @@ VLM_TEST_SETTINGS = {
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
marks=[large_gpu_mark(min_gb=32)],
|
||||
),
|
||||
"granite4_vision": VLMTestInfo(
|
||||
models=["ibm-granite/granite-vision-4.1-4b"],
|
||||
test_type=(VLMTestType.IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}\n<|assistant|>\n",
|
||||
max_model_len=8192,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=_granite4_vision_vllm_to_hf_output,
|
||||
image_size_factors=[(1.0,)],
|
||||
vllm_runner_kwargs={
|
||||
"enable_lora": True,
|
||||
"max_lora_rank": 256,
|
||||
"default_mm_loras": {"image": "ibm-granite/granite-vision-4.1-4b"},
|
||||
},
|
||||
),
|
||||
"h2ovl": VLMTestInfo(
|
||||
models=[
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
|
||||
@@ -899,6 +899,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-vision-4.1-4b",
|
||||
is_available_online=False,
|
||||
),
|
||||
"GraniteVision": _HfExamplesInfo("ibm-granite/granite-vision-3.3-2b"),
|
||||
"GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-speech-3.3-2b",
|
||||
|
||||
Reference in New Issue
Block a user