Add Cosmos3 Reasoner model (#43356)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: MaciejBalaNV <mbala@nvidia.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Roger Wang
parent
1b5437cec8
commit
9aa131f944
@@ -551,6 +551,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ |
|
||||
| `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ |
|
||||
| `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ |
|
||||
| `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + I<sup>E+</sup> + V<sup>E+</sup> | `nvidia/Cosmos3-Nano` | | ✅︎ |
|
||||
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
|
||||
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
|
||||
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -383,6 +383,26 @@ VLM_TEST_SETTINGS = {
|
||||
max_tokens=8,
|
||||
dtype="bfloat16",
|
||||
),
|
||||
"cosmos3": VLMTestInfo(
|
||||
models=["nvidia/Cosmos3-Nano"],
|
||||
test_type=(
|
||||
VLMTestType.IMAGE,
|
||||
VLMTestType.MULTI_IMAGE,
|
||||
VLMTestType.VIDEO,
|
||||
),
|
||||
enforce_eager=False,
|
||||
needs_video_metadata=True,
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
num_logprobs=20,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
|
||||
patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
|
||||
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
|
||||
),
|
||||
"deepseek_vl_v2": VLMTestInfo(
|
||||
models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
|
||||
@@ -16,6 +16,64 @@ from vllm.transformers_utils.config import try_get_safetensors_metadata
|
||||
from ..registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS
|
||||
|
||||
|
||||
def test_cosmos3_new_checkpoint_weights_mapper():
|
||||
from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration
|
||||
|
||||
mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper
|
||||
|
||||
assert mapper.apply_list(
|
||||
[
|
||||
"layers.0.self_attn.to_q.weight",
|
||||
"layers.0.self_attn.to_k.weight",
|
||||
"layers.0.self_attn.to_v.weight",
|
||||
"layers.0.self_attn.to_out.weight",
|
||||
"layers.0.self_attn.norm_q.weight",
|
||||
"layers.0.self_attn.norm_k.weight",
|
||||
"embed_tokens.weight",
|
||||
"norm.weight",
|
||||
"lm_head.weight",
|
||||
"blocks.0.attn.qkv.weight",
|
||||
]
|
||||
) == [
|
||||
"language_model.model.layers.0.self_attn.q_proj.weight",
|
||||
"language_model.model.layers.0.self_attn.k_proj.weight",
|
||||
"language_model.model.layers.0.self_attn.v_proj.weight",
|
||||
"language_model.model.layers.0.self_attn.o_proj.weight",
|
||||
"language_model.model.layers.0.self_attn.q_norm.weight",
|
||||
"language_model.model.layers.0.self_attn.k_norm.weight",
|
||||
"language_model.model.embed_tokens.weight",
|
||||
"language_model.model.norm.weight",
|
||||
"language_model.lm_head.weight",
|
||||
"visual.blocks.0.attn.qkv.weight",
|
||||
]
|
||||
|
||||
assert (
|
||||
mapper.apply_list(
|
||||
[
|
||||
"layers.0.self_attn.add_q_proj.weight",
|
||||
"layers.0.self_attn.add_k_proj.weight",
|
||||
"layers.0.self_attn.add_v_proj.weight",
|
||||
"layers.0.self_attn.to_add_out.weight",
|
||||
"layers.0.self_attn.norm_added_q.weight",
|
||||
"layers.0.self_attn.norm_added_k.weight",
|
||||
"layers.0.self_attn.q_proj_moe_gen.weight",
|
||||
"layers.0.mlp_moe_gen.gate_up_proj.weight",
|
||||
"norm_moe_gen.weight",
|
||||
"proj_in.weight",
|
||||
"proj_out.weight",
|
||||
"time_embedder.linear_1.weight",
|
||||
"audio_proj_in.weight",
|
||||
"audio_proj_out.weight",
|
||||
"action_proj_in.weight",
|
||||
"action_proj_out.weight",
|
||||
"audio_modality_embed",
|
||||
"action_modality_embed",
|
||||
]
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
||||
|
||||
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Create weights from safetensors checkpoint metadata"""
|
||||
metadata = try_get_safetensors_metadata(repo)
|
||||
|
||||
@@ -846,6 +846,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/command-a-vision-07-2025"
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Nano",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
is_available_online=False,
|
||||
),
|
||||
"DeepseekVLV2ForCausalLM": _HfExamplesInfo(
|
||||
"deepseek-ai/deepseek-vl2-tiny",
|
||||
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
|
||||
|
||||
@@ -177,7 +177,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
for pattern in allow_patterns:
|
||||
hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
|
||||
if len(hf_weights_files) > 0:
|
||||
if pattern == "*.safetensors":
|
||||
if pattern.endswith(".safetensors"):
|
||||
use_safetensors = True
|
||||
break
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import regex
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
|
||||
# alongside a generation tower in a flat key layout. This mapper drops
|
||||
# the generation tower weights and rewrites the understanding tower keys
|
||||
# into the nested form expected by Qwen3VLForConditionalGeneration.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_regex={
|
||||
regex.compile(
|
||||
r"^(layers\.|embed_tokens\.|norm\.)(.+)$"
|
||||
): r"language_model.model.\1\2",
|
||||
regex.compile(
|
||||
r"^(blocks\.|merger\.|patch_embed\.|pos_embed\.|deepstack_merger_list\.)"
|
||||
): r"visual.\1",
|
||||
regex.compile(r"^audio_modality_embed(?:\..*)?$"): None,
|
||||
regex.compile(r"^action_modality_embed(?:\..*)?$"): None,
|
||||
},
|
||||
orig_to_new_substr={
|
||||
"_moe_gen": None,
|
||||
".add_q_proj.": None,
|
||||
".add_k_proj.": None,
|
||||
".add_v_proj.": None,
|
||||
".to_add_out.": None,
|
||||
".norm_added_q.": None,
|
||||
".norm_added_k.": None,
|
||||
".to_q.": ".q_proj.",
|
||||
".to_k.": ".k_proj.",
|
||||
".to_v.": ".v_proj.",
|
||||
".to_out.": ".o_proj.",
|
||||
".norm_q.": ".q_norm.",
|
||||
".norm_k.": ".k_norm.",
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
"proj_in.": None,
|
||||
"proj_out.": None,
|
||||
"time_embedder.": None,
|
||||
"audio_proj_in.": None,
|
||||
"audio_proj_out.": None,
|
||||
"action_proj_in.": None,
|
||||
"action_proj_out.": None,
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
},
|
||||
)
|
||||
|
||||
allow_patterns_overrides = ["transformer/*.safetensors"]
|
||||
|
||||
"""
|
||||
Cosmos3 checkpoint separates transformer weights and vision_encoder weights
|
||||
into separate directories, as it's in diffusers checkpoint format.
|
||||
Using secondary_weights here to load all necessary weights for
|
||||
the Reasoner-only part.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=vllm_config.model_config.model,
|
||||
revision=vllm_config.model_config.revision,
|
||||
prefix="",
|
||||
allow_patterns_overrides=["vision_encoder/*.safetensors"],
|
||||
),
|
||||
]
|
||||
@@ -369,6 +369,7 @@ _MULTIMODAL_MODELS = {
|
||||
"cohere2_vision",
|
||||
"Cohere2VisionForConditionalGeneration",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
|
||||
@@ -94,6 +94,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
colqwen3="ColQwen3Config",
|
||||
ops_colqwen3="OpsColQwen3Config",
|
||||
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
|
||||
cosmos3_omni="Cosmos3Config",
|
||||
deepseek_vl_v2="DeepseekVLV2Config",
|
||||
deepseek_v32="DeepseekV3Config",
|
||||
deepseek_v4="DeepseekV4Config",
|
||||
|
||||
@@ -25,6 +25,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
|
||||
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
|
||||
"DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4",
|
||||
"DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr",
|
||||
@@ -95,6 +96,7 @@ __all__ = [
|
||||
"ColQwen3Config",
|
||||
"OpsColQwen3Config",
|
||||
"Qwen3VLNemotronEmbedConfig",
|
||||
"Cosmos3Config",
|
||||
"DeepseekVLV2Config",
|
||||
"DeepseekV3Config",
|
||||
"DeepseekV4Config",
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers.models.qwen3_vl.configuration_qwen3_vl import Qwen3VLConfig
|
||||
|
||||
|
||||
class Cosmos3Config(Qwen3VLConfig):
|
||||
model_type = "cosmos3_omni"
|
||||
Reference in New Issue
Block a user