Add Cosmos3 Reasoner model (#43356)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
MaciejBalaNV
2026-05-28 09:43:55 -07:00
committed by GitHub
co-authored by Isotr0py Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Roger Wang
parent 1b5437cec8
commit 9aa131f944
10 changed files with 170 additions and 1 deletions
@@ -383,6 +383,26 @@ VLM_TEST_SETTINGS = {
max_tokens=8,
dtype="bfloat16",
),
"cosmos3": VLMTestInfo(
models=["nvidia/Cosmos3-Nano"],
test_type=(
VLMTestType.IMAGE,
VLMTestType.MULTI_IMAGE,
VLMTestType.VIDEO,
),
enforce_eager=False,
needs_video_metadata=True,
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
max_model_len=4096,
max_num_seqs=2,
num_logprobs=20,
auto_cls=AutoModelForImageTextToText,
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
),
"deepseek_vl_v2": VLMTestInfo(
models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+58
View File
@@ -16,6 +16,64 @@ from vllm.transformers_utils.config import try_get_safetensors_metadata
from ..registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS
def test_cosmos3_new_checkpoint_weights_mapper():
from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration
mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper
assert mapper.apply_list(
[
"layers.0.self_attn.to_q.weight",
"layers.0.self_attn.to_k.weight",
"layers.0.self_attn.to_v.weight",
"layers.0.self_attn.to_out.weight",
"layers.0.self_attn.norm_q.weight",
"layers.0.self_attn.norm_k.weight",
"embed_tokens.weight",
"norm.weight",
"lm_head.weight",
"blocks.0.attn.qkv.weight",
]
) == [
"language_model.model.layers.0.self_attn.q_proj.weight",
"language_model.model.layers.0.self_attn.k_proj.weight",
"language_model.model.layers.0.self_attn.v_proj.weight",
"language_model.model.layers.0.self_attn.o_proj.weight",
"language_model.model.layers.0.self_attn.q_norm.weight",
"language_model.model.layers.0.self_attn.k_norm.weight",
"language_model.model.embed_tokens.weight",
"language_model.model.norm.weight",
"language_model.lm_head.weight",
"visual.blocks.0.attn.qkv.weight",
]
assert (
mapper.apply_list(
[
"layers.0.self_attn.add_q_proj.weight",
"layers.0.self_attn.add_k_proj.weight",
"layers.0.self_attn.add_v_proj.weight",
"layers.0.self_attn.to_add_out.weight",
"layers.0.self_attn.norm_added_q.weight",
"layers.0.self_attn.norm_added_k.weight",
"layers.0.self_attn.q_proj_moe_gen.weight",
"layers.0.mlp_moe_gen.gate_up_proj.weight",
"norm_moe_gen.weight",
"proj_in.weight",
"proj_out.weight",
"time_embedder.linear_1.weight",
"audio_proj_in.weight",
"audio_proj_out.weight",
"action_proj_in.weight",
"action_proj_out.weight",
"audio_modality_embed",
"action_modality_embed",
]
)
== []
)
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
metadata = try_get_safetensors_metadata(repo)
+6
View File
@@ -846,6 +846,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025"
),
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Nano",
max_model_len=4096,
min_transformers_version="4.57",
is_available_online=False,
),
"DeepseekVLV2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/deepseek-vl2-tiny",
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},