From 9aa131f9444649b3036d811d5bdb26c75f8d4157 Mon Sep 17 00:00:00 2001 From: MaciejBalaNV Date: Thu, 28 May 2026 18:43:55 +0200 Subject: [PATCH] Add Cosmos3 Reasoner model (#43356) Signed-off-by: Maciej Bala Signed-off-by: MaciejBalaNV Co-authored-by: Isotr0py Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Roger Wang --- docs/models/supported_models.md | 1 + .../multimodal/generation/test_common.py | 20 +++++ tests/models/multimodal/test_mapping.py | 58 +++++++++++++++ tests/models/registry.py | 6 ++ .../model_loader/default_loader.py | 2 +- vllm/model_executor/models/cosmos3.py | 73 +++++++++++++++++++ vllm/model_executor/models/registry.py | 1 + vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 2 + vllm/transformers_utils/configs/cosmos3.py | 7 ++ 10 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 vllm/model_executor/models/cosmos3.py create mode 100644 vllm/transformers_utils/configs/cosmos3.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 599da3c6b5..1f38200a78 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -551,6 +551,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ | | `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ | | `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I+ | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ | +| `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Nano` | | ✅︎ | | `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I+ | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ | | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I+ | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ | | `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I+ | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index 88267fc6d6..6160280993 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -383,6 +383,26 @@ VLM_TEST_SETTINGS = { max_tokens=8, dtype="bfloat16", ), + "cosmos3": VLMTestInfo( + models=["nvidia/Cosmos3-Nano"], + test_type=( + VLMTestType.IMAGE, + VLMTestType.MULTI_IMAGE, + VLMTestType.VIDEO, + ), + enforce_eager=False, + needs_video_metadata=True, + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501 + max_model_len=4096, + max_num_seqs=2, + num_logprobs=20, + auto_cls=AutoModelForImageTextToText, + vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output, + patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner, + image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)], + ), "deepseek_vl_v2": VLMTestInfo( models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), diff --git a/tests/models/multimodal/test_mapping.py b/tests/models/multimodal/test_mapping.py index f866d467d0..b1081726f2 100644 --- a/tests/models/multimodal/test_mapping.py +++ b/tests/models/multimodal/test_mapping.py @@ -16,6 +16,64 @@ from vllm.transformers_utils.config import try_get_safetensors_metadata from ..registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS +def test_cosmos3_new_checkpoint_weights_mapper(): + from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration + + mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper + + assert mapper.apply_list( + [ + "layers.0.self_attn.to_q.weight", + "layers.0.self_attn.to_k.weight", + "layers.0.self_attn.to_v.weight", + "layers.0.self_attn.to_out.weight", + "layers.0.self_attn.norm_q.weight", + "layers.0.self_attn.norm_k.weight", + "embed_tokens.weight", + "norm.weight", + "lm_head.weight", + "blocks.0.attn.qkv.weight", + ] + ) == [ + "language_model.model.layers.0.self_attn.q_proj.weight", + "language_model.model.layers.0.self_attn.k_proj.weight", + "language_model.model.layers.0.self_attn.v_proj.weight", + "language_model.model.layers.0.self_attn.o_proj.weight", + "language_model.model.layers.0.self_attn.q_norm.weight", + "language_model.model.layers.0.self_attn.k_norm.weight", + "language_model.model.embed_tokens.weight", + "language_model.model.norm.weight", + "language_model.lm_head.weight", + "visual.blocks.0.attn.qkv.weight", + ] + + assert ( + mapper.apply_list( + [ + "layers.0.self_attn.add_q_proj.weight", + "layers.0.self_attn.add_k_proj.weight", + "layers.0.self_attn.add_v_proj.weight", + "layers.0.self_attn.to_add_out.weight", + "layers.0.self_attn.norm_added_q.weight", + "layers.0.self_attn.norm_added_k.weight", + "layers.0.self_attn.q_proj_moe_gen.weight", + "layers.0.mlp_moe_gen.gate_up_proj.weight", + "norm_moe_gen.weight", + "proj_in.weight", + "proj_out.weight", + "time_embedder.linear_1.weight", + "audio_proj_in.weight", + "audio_proj_out.weight", + "action_proj_in.weight", + "action_proj_out.weight", + "audio_modality_embed", + "action_modality_embed", + ] + ) + == [] + ) + + def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]: """Create weights from safetensors checkpoint metadata""" metadata = try_get_safetensors_metadata(repo) diff --git a/tests/models/registry.py b/tests/models/registry.py index 356d06d94e..226eb80aca 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -846,6 +846,12 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Cohere2VisionForConditionalGeneration": _HfExamplesInfo( "CohereLabs/command-a-vision-07-2025" ), + "Cosmos3ForConditionalGeneration": _HfExamplesInfo( + "nvidia/Cosmos3-Nano", + max_model_len=4096, + min_transformers_version="4.57", + is_available_online=False, + ), "DeepseekVLV2ForCausalLM": _HfExamplesInfo( "deepseek-ai/deepseek-vl2-tiny", extras={"fork": "Isotr0py/deepseek-vl2-tiny"}, diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index 5b5632e319..43d5d4a449 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -177,7 +177,7 @@ class DefaultModelLoader(BaseModelLoader): for pattern in allow_patterns: hf_weights_files += glob.glob(os.path.join(hf_folder, pattern)) if len(hf_weights_files) > 0: - if pattern == "*.safetensors": + if pattern.endswith(".safetensors"): use_safetensors = True break diff --git a/vllm/model_executor/models/cosmos3.py b/vllm/model_executor/models/cosmos3.py new file mode 100644 index 0000000000..e0ba8a247d --- /dev/null +++ b/vllm/model_executor/models/cosmos3.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import regex + +from vllm.config import VllmConfig +from vllm.model_executor.model_loader.default_loader import DefaultModelLoader +from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration +from vllm.model_executor.models.utils import WeightsMapper + + +class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration): + # Cosmos3 unified checkpoints store a Qwen3-VL understanding tower + # alongside a generation tower in a flat key layout. This mapper drops + # the generation tower weights and rewrites the understanding tower keys + # into the nested form expected by Qwen3VLForConditionalGeneration. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_regex={ + regex.compile( + r"^(layers\.|embed_tokens\.|norm\.)(.+)$" + ): r"language_model.model.\1\2", + regex.compile( + r"^(blocks\.|merger\.|patch_embed\.|pos_embed\.|deepstack_merger_list\.)" + ): r"visual.\1", + regex.compile(r"^audio_modality_embed(?:\..*)?$"): None, + regex.compile(r"^action_modality_embed(?:\..*)?$"): None, + }, + orig_to_new_substr={ + "_moe_gen": None, + ".add_q_proj.": None, + ".add_k_proj.": None, + ".add_v_proj.": None, + ".to_add_out.": None, + ".norm_added_q.": None, + ".norm_added_k.": None, + ".to_q.": ".q_proj.", + ".to_k.": ".k_proj.", + ".to_v.": ".v_proj.", + ".to_out.": ".o_proj.", + ".norm_q.": ".q_norm.", + ".norm_k.": ".k_norm.", + }, + orig_to_new_prefix={ + "proj_in.": None, + "proj_out.": None, + "time_embedder.": None, + "audio_proj_in.": None, + "audio_proj_out.": None, + "action_proj_in.": None, + "action_proj_out.": None, + "lm_head.": "language_model.lm_head.", + }, + ) + + allow_patterns_overrides = ["transformer/*.safetensors"] + + """ + Cosmos3 checkpoint separates transformer weights and vision_encoder weights + into separate directories, as it's in diffusers checkpoint format. + Using secondary_weights here to load all necessary weights for + the Reasoner-only part. + """ + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__(vllm_config=vllm_config, prefix=prefix) + self.secondary_weights = [ + DefaultModelLoader.Source( + model_or_path=vllm_config.model_config.model, + revision=vllm_config.model_config.revision, + prefix="", + allow_patterns_overrides=["vision_encoder/*.safetensors"], + ), + ] diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 8c358ba924..5e6f106150 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -369,6 +369,7 @@ _MULTIMODAL_MODELS = { "cohere2_vision", "Cohere2VisionForConditionalGeneration", ), + "Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"), "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"), "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"), "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"), diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 49f0b82530..0969d81690 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -94,6 +94,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( colqwen3="ColQwen3Config", ops_colqwen3="OpsColQwen3Config", qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig", + cosmos3_omni="Cosmos3Config", deepseek_vl_v2="DeepseekVLV2Config", deepseek_v32="DeepseekV3Config", deepseek_v4="DeepseekV4Config", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index e98f0ae055..4a90655dcc 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -25,6 +25,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "ColQwen3Config": "vllm.transformers_utils.configs.colqwen3", "OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3", "Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3", + "Cosmos3Config": "vllm.transformers_utils.configs.cosmos3", "DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2", "DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4", "DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr", @@ -95,6 +96,7 @@ __all__ = [ "ColQwen3Config", "OpsColQwen3Config", "Qwen3VLNemotronEmbedConfig", + "Cosmos3Config", "DeepseekVLV2Config", "DeepseekV3Config", "DeepseekV4Config", diff --git a/vllm/transformers_utils/configs/cosmos3.py b/vllm/transformers_utils/configs/cosmos3.py new file mode 100644 index 0000000000..965801888a --- /dev/null +++ b/vllm/transformers_utils/configs/cosmos3.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLConfig + + +class Cosmos3Config(Qwen3VLConfig): + model_type = "cosmos3_omni"