Add Cosmos3 Reasoner model (#43356)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
MaciejBalaNV
2026-05-28 09:43:55 -07:00
committed by GitHub
co-authored by Isotr0py Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Roger Wang
parent 1b5437cec8
commit 9aa131f944
10 changed files with 170 additions and 1 deletions
+73
View File
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import regex
from vllm.config import VllmConfig
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm.model_executor.models.utils import WeightsMapper
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
# alongside a generation tower in a flat key layout. This mapper drops
# the generation tower weights and rewrites the understanding tower keys
# into the nested form expected by Qwen3VLForConditionalGeneration.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_regex={
regex.compile(
r"^(layers\.|embed_tokens\.|norm\.)(.+)$"
): r"language_model.model.\1\2",
regex.compile(
r"^(blocks\.|merger\.|patch_embed\.|pos_embed\.|deepstack_merger_list\.)"
): r"visual.\1",
regex.compile(r"^audio_modality_embed(?:\..*)?$"): None,
regex.compile(r"^action_modality_embed(?:\..*)?$"): None,
},
orig_to_new_substr={
"_moe_gen": None,
".add_q_proj.": None,
".add_k_proj.": None,
".add_v_proj.": None,
".to_add_out.": None,
".norm_added_q.": None,
".norm_added_k.": None,
".to_q.": ".q_proj.",
".to_k.": ".k_proj.",
".to_v.": ".v_proj.",
".to_out.": ".o_proj.",
".norm_q.": ".q_norm.",
".norm_k.": ".k_norm.",
},
orig_to_new_prefix={
"proj_in.": None,
"proj_out.": None,
"time_embedder.": None,
"audio_proj_in.": None,
"audio_proj_out.": None,
"action_proj_in.": None,
"action_proj_out.": None,
"lm_head.": "language_model.lm_head.",
},
)
allow_patterns_overrides = ["transformer/*.safetensors"]
"""
Cosmos3 checkpoint separates transformer weights and vision_encoder weights
into separate directories, as it's in diffusers checkpoint format.
Using secondary_weights here to load all necessary weights for
the Reasoner-only part.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=vllm_config.model_config.model,
revision=vllm_config.model_config.revision,
prefix="",
allow_patterns_overrides=["vision_encoder/*.safetensors"],
),
]
+1
View File
@@ -369,6 +369,7 @@ _MULTIMODAL_MODELS = {
"cohere2_vision",
"Cohere2VisionForConditionalGeneration",
),
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),