Add Molmo2 multimodal model support (#30997)

Signed-off-by: sanghol <sanghol@allenai.org>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
sangho.lee
2026-01-14 15:33:09 +08:00
committed by GitHub
co-authored by Isotr0py
parent 9312a6c03a
commit 7e6f123810
11 changed files with 3215 additions and 1 deletions
@@ -123,6 +123,7 @@ MM_DATA_PATCHES = {
"glm4v": glm4_1v_patch_mm_data,
"glm4v_moe": glm4_1v_patch_mm_data,
"glmasr": glmasr_patch_mm_data,
"molmo2": qwen3_vl_patch_mm_data,
"qwen3_vl": qwen3_vl_patch_mm_data,
"qwen3_vl_moe": qwen3_vl_patch_mm_data,
}
+13
View File
@@ -92,6 +92,11 @@ class _HfExamplesInfo:
length that is too large to fit into memory in CI.
"""
max_num_batched_tokens: int | None = None
"""
The maximum number of tokens to be processed in a single batch.
"""
revision: str | None = None
"""
The specific revision (commit hash, tag, or branch) to use for the model.
@@ -817,6 +822,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
extras={"olmo": "allenai/Molmo-7B-O-0924"},
trust_remote_code=True,
),
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
"allenai/Molmo2-8B",
extras={"olmo": "allenai/Molmo2-O-7B"},
min_transformers_version="4.51",
trust_remote_code=True,
# required by current PrefixLM implementation
max_num_batched_tokens=31872,
),
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
"Llama_Nemotron_Nano_VL": _HfExamplesInfo(
"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
+1
View File
@@ -140,6 +140,7 @@ def can_initialize(
else None,
trust_remote_code=model_info.trust_remote_code,
max_model_len=model_info.max_model_len,
max_num_batched_tokens=model_info.max_num_batched_tokens,
# these tests seem to produce leftover memory
gpu_memory_utilization=0.80,
load_format="dummy",