Add Molmo2 multimodal model support (#30997)
Signed-off-by: sanghol <sanghol@allenai.org> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -123,6 +123,7 @@ MM_DATA_PATCHES = {
|
||||
"glm4v": glm4_1v_patch_mm_data,
|
||||
"glm4v_moe": glm4_1v_patch_mm_data,
|
||||
"glmasr": glmasr_patch_mm_data,
|
||||
"molmo2": qwen3_vl_patch_mm_data,
|
||||
"qwen3_vl": qwen3_vl_patch_mm_data,
|
||||
"qwen3_vl_moe": qwen3_vl_patch_mm_data,
|
||||
}
|
||||
|
||||
@@ -92,6 +92,11 @@ class _HfExamplesInfo:
|
||||
length that is too large to fit into memory in CI.
|
||||
"""
|
||||
|
||||
max_num_batched_tokens: int | None = None
|
||||
"""
|
||||
The maximum number of tokens to be processed in a single batch.
|
||||
"""
|
||||
|
||||
revision: str | None = None
|
||||
"""
|
||||
The specific revision (commit hash, tag, or branch) to use for the model.
|
||||
@@ -817,6 +822,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
extras={"olmo": "allenai/Molmo-7B-O-0924"},
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allenai/Molmo2-8B",
|
||||
extras={"olmo": "allenai/Molmo2-O-7B"},
|
||||
min_transformers_version="4.51",
|
||||
trust_remote_code=True,
|
||||
# required by current PrefixLM implementation
|
||||
max_num_batched_tokens=31872,
|
||||
),
|
||||
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
|
||||
"Llama_Nemotron_Nano_VL": _HfExamplesInfo(
|
||||
"nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
|
||||
|
||||
@@ -140,6 +140,7 @@ def can_initialize(
|
||||
else None,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
max_model_len=model_info.max_model_len,
|
||||
max_num_batched_tokens=model_info.max_num_batched_tokens,
|
||||
# these tests seem to produce leftover memory
|
||||
gpu_memory_utilization=0.80,
|
||||
load_format="dummy",
|
||||
|
||||
Reference in New Issue
Block a user