Add Cosmos3 Reasoner model (#43356)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: MaciejBalaNV <mbala@nvidia.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Roger Wang
parent
1b5437cec8
commit
9aa131f944
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import regex
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
|
||||
# alongside a generation tower in a flat key layout. This mapper drops
|
||||
# the generation tower weights and rewrites the understanding tower keys
|
||||
# into the nested form expected by Qwen3VLForConditionalGeneration.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_regex={
|
||||
regex.compile(
|
||||
r"^(layers\.|embed_tokens\.|norm\.)(.+)$"
|
||||
): r"language_model.model.\1\2",
|
||||
regex.compile(
|
||||
r"^(blocks\.|merger\.|patch_embed\.|pos_embed\.|deepstack_merger_list\.)"
|
||||
): r"visual.\1",
|
||||
regex.compile(r"^audio_modality_embed(?:\..*)?$"): None,
|
||||
regex.compile(r"^action_modality_embed(?:\..*)?$"): None,
|
||||
},
|
||||
orig_to_new_substr={
|
||||
"_moe_gen": None,
|
||||
".add_q_proj.": None,
|
||||
".add_k_proj.": None,
|
||||
".add_v_proj.": None,
|
||||
".to_add_out.": None,
|
||||
".norm_added_q.": None,
|
||||
".norm_added_k.": None,
|
||||
".to_q.": ".q_proj.",
|
||||
".to_k.": ".k_proj.",
|
||||
".to_v.": ".v_proj.",
|
||||
".to_out.": ".o_proj.",
|
||||
".norm_q.": ".q_norm.",
|
||||
".norm_k.": ".k_norm.",
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
"proj_in.": None,
|
||||
"proj_out.": None,
|
||||
"time_embedder.": None,
|
||||
"audio_proj_in.": None,
|
||||
"audio_proj_out.": None,
|
||||
"action_proj_in.": None,
|
||||
"action_proj_out.": None,
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
},
|
||||
)
|
||||
|
||||
allow_patterns_overrides = ["transformer/*.safetensors"]
|
||||
|
||||
"""
|
||||
Cosmos3 checkpoint separates transformer weights and vision_encoder weights
|
||||
into separate directories, as it's in diffusers checkpoint format.
|
||||
Using secondary_weights here to load all necessary weights for
|
||||
the Reasoner-only part.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=vllm_config.model_config.model,
|
||||
revision=vllm_config.model_config.revision,
|
||||
prefix="",
|
||||
allow_patterns_overrides=["vision_encoder/*.safetensors"],
|
||||
),
|
||||
]
|
||||
@@ -369,6 +369,7 @@ _MULTIMODAL_MODELS = {
|
||||
"cohere2_vision",
|
||||
"Cohere2VisionForConditionalGeneration",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
|
||||
Reference in New Issue
Block a user