Add Cosmos3 Edge Reasoner model (#48291)

Signed-off-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
Co-authored-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
This commit is contained in:
adsridhar
2026-07-14 08:14:50 +00:00
committed by GitHub
co-authored by Bartosz Stefaniak
parent c9a788eedc
commit 894ebb27f5
13 changed files with 1474 additions and 27 deletions
@@ -0,0 +1,101 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from vllm.assets.video import VideoAsset
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
MODEL_ID = "nvidia/Cosmos3-Edge"
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
@pytest.fixture(scope="module")
def processor():
if LOCAL_MODEL_PATH:
model_config = ModelConfig(
LOCAL_MODEL_PATH,
tokenizer=LOCAL_MODEL_PATH,
max_model_len=4096,
limit_mm_per_prompt={"image": 2, "video": 1},
)
else:
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 2, "video": 1},
)
model_config = ctx.model_config
return MULTIMODAL_REGISTRY.create_processor(model_config)
def _assert_image_outputs(processor, processed, num_images: int) -> None:
mm_data = processed["mm_kwargs"].get_data()
grid_thw = mm_data["image_grid_thw"]
pixel_values = mm_data["pixel_values"]
assert grid_thw.shape == (num_images, 3)
assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
image_placeholders = processed["mm_placeholders"]["image"]
assert len(image_placeholders) == num_images
assert [placeholder.length for placeholder in image_placeholders] == (
expected_tokens
)
image_token_id = processor.info.get_hf_processor().image_token_id
assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
def _assert_video_outputs(processor, processed) -> None:
mm_data = processed["mm_kwargs"].get_data()
grid_thw = mm_data["video_grid_thw"]
pixel_values = mm_data["pixel_values_videos"]
assert grid_thw.shape == (1, 3)
assert pixel_values.shape[0] == int(grid_thw.prod())
assert len(processed["mm_placeholders"]["video"]) == 1
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
expected_tokens = int(grid_thw.prod()) // merge_size**2
video_token_id = processor.info.get_hf_config().video_token_id
assert processed["prompt_token_ids"].count(video_token_id) == expected_tokens
@pytest.mark.parametrize("num_images", [1, 2])
def test_process_images(
processor,
image_assets: ImageTestAssets,
num_images: int,
) -> None:
images = [asset.pil_image for asset in image_assets[:num_images]]
processed = processor(
IMAGE_PLACEHOLDER * num_images,
mm_items=processor.info.parse_mm_data({"image": images}),
hf_processor_mm_kwargs={},
)
_assert_image_outputs(processor, processed, num_images)
def test_process_video(processor) -> None:
video_asset = VideoAsset(name="baby_reading", num_frames=8)
video = (video_asset.np_ndarrays, video_asset.metadata)
processed = processor(
VIDEO_PLACEHOLDER,
mm_items=processor.info.parse_mm_data({"video": [video]}),
hf_processor_mm_kwargs={},
)
_assert_video_outputs(processor, processed)
+71
View File
@@ -74,6 +74,77 @@ def test_cosmos3_new_checkpoint_weights_mapper():
)
def test_cosmos3_edge_checkpoint_weights_mapper():
from vllm.model_executor.models.cosmos3_edge import (
Cosmos3EdgeForConditionalGeneration,
)
mapper = Cosmos3EdgeForConditionalGeneration.hf_to_vllm_mapper
assert mapper.apply_list(
[
"embed_tokens.weight",
"norm.weight",
"layers.0.input_layernorm.weight",
"layers.0.self_attn.to_q.weight",
"layers.0.self_attn.to_k.weight",
"layers.0.self_attn.to_v.weight",
"layers.0.self_attn.to_out.weight",
"layers.0.post_attention_layernorm.weight",
"layers.0.mlp.up_proj.weight",
"layers.0.mlp.down_proj.weight",
"layers.27.input_layernorm.weight",
"layers.27.self_attn.to_q.weight",
"layers.27.post_attention_layernorm.weight",
"layers.27.mlp.down_proj.weight",
"model.visual.embeddings.patch_embedding.weight",
"model.visual.encoder.layers.0.self_attn.q_proj.weight",
"model.projector.linear_fc1.weight",
"lm_head.weight",
]
) == [
"language_model.model.embed_tokens.weight",
"language_model.model.norm_f.weight",
"language_model.model.layers.0.norm.weight",
"language_model.model.layers.0.mixer.qkv_proj.weight",
"language_model.model.layers.0.mixer.qkv_proj.weight",
"language_model.model.layers.0.mixer.qkv_proj.weight",
"language_model.model.layers.0.mixer.o_proj.weight",
"language_model.model.layers.1.norm.weight",
"language_model.model.layers.1.mixer.up_proj.weight",
"language_model.model.layers.1.mixer.down_proj.weight",
"language_model.model.layers.54.norm.weight",
"language_model.model.layers.54.mixer.qkv_proj.weight",
"language_model.model.layers.55.norm.weight",
"language_model.model.layers.55.mixer.down_proj.weight",
"visual.encoder.embeddings.patch_embedding.weight",
"visual.encoder.encoder.layers.0.self_attn.qkv_proj.weight",
"visual.projector.linear_fc1.weight",
"language_model.lm_head.weight",
]
assert (
mapper.apply_list(
[
"layers.0.self_attn.add_q_proj.weight",
"layers.0.self_attn.add_k_proj.weight",
"layers.0.self_attn.add_v_proj.weight",
"layers.0.self_attn.to_add_out.weight",
"layers.0.self_attn.norm_added_q.weight",
"layers.0.self_attn.norm_added_k.weight",
"layers.0.self_attn.q_proj_moe_gen.weight",
"layers.0.mlp_moe_gen.up_proj.weight",
"norm_moe_gen.weight",
"proj_in.weight",
"proj_out.weight",
"audio_modality_embed",
"action_modality_embed",
]
)
== []
)
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
metadata = try_get_safetensors_metadata(repo)
+5
View File
@@ -811,6 +811,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
max_model_len=4096,
min_transformers_version="4.57",
),
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Edge",
max_model_len=4096,
is_available_online=False,
),
"DeepseekVLV2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/deepseek-vl2-tiny",
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},