From 894ebb27f5c7d7f8758266163ffd503132e432d2 Mon Sep 17 00:00:00 2001 From: adsridhar Date: Tue, 14 Jul 2026 01:14:50 -0700 Subject: [PATCH] Add Cosmos3 Edge Reasoner model (#48291) Signed-off-by: Bartosz Stefaniak Co-authored-by: Bartosz Stefaniak --- docs/models/supported_models.md | 1 + .../processing/test_cosmos3_edge.py | 101 +++ tests/models/multimodal/test_mapping.py | 71 ++ tests/models/registry.py | 5 + vllm/model_executor/models/cosmos3.py | 25 +- vllm/model_executor/models/cosmos3_edge.py | 751 ++++++++++++++++++ vllm/model_executor/models/nemotron_h.py | 30 +- vllm/model_executor/models/registry.py | 4 + vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 8 + .../configs/cosmos3_edge.py | 146 ++++ .../transformers_utils/processors/__init__.py | 2 + .../processors/cosmos3_edge.py | 356 +++++++++ 13 files changed, 1474 insertions(+), 27 deletions(-) create mode 100644 tests/models/multimodal/processing/test_cosmos3_edge.py create mode 100644 vllm/model_executor/models/cosmos3_edge.py create mode 100644 vllm/transformers_utils/configs/cosmos3_edge.py create mode 100644 vllm/transformers_utils/processors/cosmos3_edge.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index f19f01c698..37d27234aa 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -514,6 +514,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ | | `Cohere2VisionForConditionalGeneration` | Command A Vision, Command-A+ | T + I+ | `CohereLabs/command-a-vision-07-2025`, `CohereLabs/command-a-plus-05-2026`, etc. | | ✅︎ | | `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | | ✅︎ | +| `Cosmos3EdgeForConditionalGeneration` | Cosmos3-Edge (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Edge` | | ✅︎ | | `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I+ | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ | | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I+ | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ | | `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I+ | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/multimodal/processing/test_cosmos3_edge.py b/tests/models/multimodal/processing/test_cosmos3_edge.py new file mode 100644 index 0000000000..5ebef08c10 --- /dev/null +++ b/tests/models/multimodal/processing/test_cosmos3_edge.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import os + +import pytest + +from vllm.assets.video import VideoAsset +from vllm.config import ModelConfig +from vllm.multimodal import MULTIMODAL_REGISTRY + +from ....conftest import ImageTestAssets +from ...utils import build_model_context + +MODEL_ID = "nvidia/Cosmos3-Edge" +IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>" +VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>" +LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH") + + +@pytest.fixture(scope="module") +def processor(): + if LOCAL_MODEL_PATH: + model_config = ModelConfig( + LOCAL_MODEL_PATH, + tokenizer=LOCAL_MODEL_PATH, + max_model_len=4096, + limit_mm_per_prompt={"image": 2, "video": 1}, + ) + else: + ctx = build_model_context( + MODEL_ID, + limit_mm_per_prompt={"image": 2, "video": 1}, + ) + model_config = ctx.model_config + + return MULTIMODAL_REGISTRY.create_processor(model_config) + + +def _assert_image_outputs(processor, processed, num_images: int) -> None: + mm_data = processed["mm_kwargs"].get_data() + grid_thw = mm_data["image_grid_thw"] + pixel_values = mm_data["pixel_values"] + + assert grid_thw.shape == (num_images, 3) + assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum()) + + merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size + expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist() + image_placeholders = processed["mm_placeholders"]["image"] + + assert len(image_placeholders) == num_images + assert [placeholder.length for placeholder in image_placeholders] == ( + expected_tokens + ) + + image_token_id = processor.info.get_hf_processor().image_token_id + assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens) + + +def _assert_video_outputs(processor, processed) -> None: + mm_data = processed["mm_kwargs"].get_data() + grid_thw = mm_data["video_grid_thw"] + pixel_values = mm_data["pixel_values_videos"] + + assert grid_thw.shape == (1, 3) + assert pixel_values.shape[0] == int(grid_thw.prod()) + assert len(processed["mm_placeholders"]["video"]) == 1 + + merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size + expected_tokens = int(grid_thw.prod()) // merge_size**2 + video_token_id = processor.info.get_hf_config().video_token_id + assert processed["prompt_token_ids"].count(video_token_id) == expected_tokens + + +@pytest.mark.parametrize("num_images", [1, 2]) +def test_process_images( + processor, + image_assets: ImageTestAssets, + num_images: int, +) -> None: + images = [asset.pil_image for asset in image_assets[:num_images]] + processed = processor( + IMAGE_PLACEHOLDER * num_images, + mm_items=processor.info.parse_mm_data({"image": images}), + hf_processor_mm_kwargs={}, + ) + + _assert_image_outputs(processor, processed, num_images) + + +def test_process_video(processor) -> None: + video_asset = VideoAsset(name="baby_reading", num_frames=8) + video = (video_asset.np_ndarrays, video_asset.metadata) + processed = processor( + VIDEO_PLACEHOLDER, + mm_items=processor.info.parse_mm_data({"video": [video]}), + hf_processor_mm_kwargs={}, + ) + + _assert_video_outputs(processor, processed) diff --git a/tests/models/multimodal/test_mapping.py b/tests/models/multimodal/test_mapping.py index b1081726f2..ce9b8a9d45 100644 --- a/tests/models/multimodal/test_mapping.py +++ b/tests/models/multimodal/test_mapping.py @@ -74,6 +74,77 @@ def test_cosmos3_new_checkpoint_weights_mapper(): ) +def test_cosmos3_edge_checkpoint_weights_mapper(): + from vllm.model_executor.models.cosmos3_edge import ( + Cosmos3EdgeForConditionalGeneration, + ) + + mapper = Cosmos3EdgeForConditionalGeneration.hf_to_vllm_mapper + + assert mapper.apply_list( + [ + "embed_tokens.weight", + "norm.weight", + "layers.0.input_layernorm.weight", + "layers.0.self_attn.to_q.weight", + "layers.0.self_attn.to_k.weight", + "layers.0.self_attn.to_v.weight", + "layers.0.self_attn.to_out.weight", + "layers.0.post_attention_layernorm.weight", + "layers.0.mlp.up_proj.weight", + "layers.0.mlp.down_proj.weight", + "layers.27.input_layernorm.weight", + "layers.27.self_attn.to_q.weight", + "layers.27.post_attention_layernorm.weight", + "layers.27.mlp.down_proj.weight", + "model.visual.embeddings.patch_embedding.weight", + "model.visual.encoder.layers.0.self_attn.q_proj.weight", + "model.projector.linear_fc1.weight", + "lm_head.weight", + ] + ) == [ + "language_model.model.embed_tokens.weight", + "language_model.model.norm_f.weight", + "language_model.model.layers.0.norm.weight", + "language_model.model.layers.0.mixer.qkv_proj.weight", + "language_model.model.layers.0.mixer.qkv_proj.weight", + "language_model.model.layers.0.mixer.qkv_proj.weight", + "language_model.model.layers.0.mixer.o_proj.weight", + "language_model.model.layers.1.norm.weight", + "language_model.model.layers.1.mixer.up_proj.weight", + "language_model.model.layers.1.mixer.down_proj.weight", + "language_model.model.layers.54.norm.weight", + "language_model.model.layers.54.mixer.qkv_proj.weight", + "language_model.model.layers.55.norm.weight", + "language_model.model.layers.55.mixer.down_proj.weight", + "visual.encoder.embeddings.patch_embedding.weight", + "visual.encoder.encoder.layers.0.self_attn.qkv_proj.weight", + "visual.projector.linear_fc1.weight", + "language_model.lm_head.weight", + ] + + assert ( + mapper.apply_list( + [ + "layers.0.self_attn.add_q_proj.weight", + "layers.0.self_attn.add_k_proj.weight", + "layers.0.self_attn.add_v_proj.weight", + "layers.0.self_attn.to_add_out.weight", + "layers.0.self_attn.norm_added_q.weight", + "layers.0.self_attn.norm_added_k.weight", + "layers.0.self_attn.q_proj_moe_gen.weight", + "layers.0.mlp_moe_gen.up_proj.weight", + "norm_moe_gen.weight", + "proj_in.weight", + "proj_out.weight", + "audio_modality_embed", + "action_modality_embed", + ] + ) + == [] + ) + + def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]: """Create weights from safetensors checkpoint metadata""" metadata = try_get_safetensors_metadata(repo) diff --git a/tests/models/registry.py b/tests/models/registry.py index 4157d23984..4e8fe96fdb 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -811,6 +811,11 @@ _MULTIMODAL_EXAMPLE_MODELS = { max_model_len=4096, min_transformers_version="4.57", ), + "Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo( + "nvidia/Cosmos3-Edge", + max_model_len=4096, + is_available_online=False, + ), "DeepseekVLV2ForCausalLM": _HfExamplesInfo( "deepseek-ai/deepseek-vl2-tiny", extras={"fork": "Isotr0py/deepseek-vl2-tiny"}, diff --git a/vllm/model_executor/models/cosmos3.py b/vllm/model_executor/models/cosmos3.py index e0ba8a247d..71629196a7 100644 --- a/vllm/model_executor/models/cosmos3.py +++ b/vllm/model_executor/models/cosmos3.py @@ -3,8 +3,6 @@ import regex -from vllm.config import VllmConfig -from vllm.model_executor.model_loader.default_loader import DefaultModelLoader from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration from vllm.model_executor.models.utils import WeightsMapper @@ -52,22 +50,7 @@ class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration): }, ) - allow_patterns_overrides = ["transformer/*.safetensors"] - - """ - Cosmos3 checkpoint separates transformer weights and vision_encoder weights - into separate directories, as it's in diffusers checkpoint format. - Using secondary_weights here to load all necessary weights for - the Reasoner-only part. - """ - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - super().__init__(vllm_config=vllm_config, prefix=prefix) - self.secondary_weights = [ - DefaultModelLoader.Source( - model_or_path=vllm_config.model_config.model, - revision=vllm_config.model_config.revision, - prefix="", - allow_patterns_overrides=["vision_encoder/*.safetensors"], - ), - ] + # Cosmos3 unified Diffusers checkpoints store reasoner weights across + # transformer/ and vision_encoder/. Match both while excluding VAE and + # sound_tokenizer weights; the mapper drops generation-only tensors. + allow_patterns_overrides = ["[tv]*er/*.safetensors"] diff --git a/vllm/model_executor/models/cosmos3_edge.py b/vllm/model_executor/models/cosmos3_edge.py new file mode 100644 index 0000000000..be61d564cc --- /dev/null +++ b/vllm/model_executor/models/cosmos3_edge.py @@ -0,0 +1,751 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable + +import torch +import torch.nn as nn +from transformers import ProcessorMixin + +from vllm.config import VllmConfig +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.models.module_mapping import MultiModelKeys +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig + +from .interfaces import ( + MultiModalEmbeddings, + SupportsMRoPE, + SupportsMultiModal, + SupportsPP, +) +from .lfm2_siglip2 import Siglip2VisionTransformer +from .nemotron_h import ( + NemotronHAttention, + NemotronHMLPDecoderLayer, + NemotronHModel, +) +from .qwen2_5_vl import ( + Qwen2_5_VLImageEmbeddingInputs, + Qwen2_5_VLImageInputs, + Qwen2_5_VLImagePixelInputs, + Qwen2_5_VLVideoEmbeddingInputs, + Qwen2_5_VLVideoInputs, + Qwen2_5_VLVideoPixelInputs, +) +from .qwen3_vl import ( + Qwen3VLDummyInputsBuilder, + Qwen3VLForConditionalGeneration, + Qwen3VLMultiModalProcessor, + Qwen3VLProcessingInfo, +) +from .utils import ( + AutoWeightsLoader, + WeightsMapper, + maybe_prefix, +) +from .vision import ( + is_vit_use_data_parallel, + run_dp_sharded_mrope_vision_model, +) + + +class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer): + """Adapts Cosmos (T, H, W) metadata to vLLM packed SigLIP2.""" + + @property + def dtype(self) -> torch.dtype: + return self.embeddings.patch_embedding.weight.dtype + + def forward( + self, + pixel_values: torch.Tensor, + grid_thw: torch.Tensor, + ) -> torch.Tensor: + # SigLIP2 attends to each frame independently, so expand every THW + # entry into T separate HW attention sequences. + grid_thw_cpu = grid_thw.to(device="cpu") + spatial_shapes = torch.repeat_interleave( + grid_thw_cpu[:, 1:], + grid_thw_cpu[:, 0], + dim=0, + ) + lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32) + lengths = lengths_cpu.to( + device=pixel_values.device, + non_blocking=True, + ) + + cu_seqlens = torch.zeros( + lengths.numel() + 1, + dtype=torch.int32, + device=pixel_values.device, + ) + cu_seqlens[1:] = lengths.cumsum(dim=0) + + max_seqlen = lengths_cpu.max().reshape(1) + + return super().forward( + pixel_values_packed=pixel_values, + spatial_shapes=spatial_shapes, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + )[0] + + +def patch_merging_by_param( + image_embeds: torch.Tensor, + grid_thw: torch.Tensor, + merge_size: int = 2, +) -> torch.Tensor: + """Merge each spatial ``merge_size`` block into the channel dimension.""" + merged_embeds: list[torch.Tensor] = [] + current_idx = 0 + hidden_size = image_embeds.shape[-1] + + for grid in grid_thw: + t, h, w = grid.tolist() + if h % merge_size != 0 or w % merge_size != 0: + raise ValueError( + "Grid height and width must be divisible by merge_size: " + f"got grid={(t, h, w)} and merge_size={merge_size}." + ) + + num_patches = t * h * w + media_embeds = image_embeds[current_idx : current_idx + num_patches] + if media_embeds.shape[0] != num_patches: + raise ValueError( + "image_embeds contains fewer patches than grid_thw describes." + ) + current_idx += num_patches + + media_embeds = media_embeds.view(t, h, w, hidden_size) + media_embeds = media_embeds.view( + t, + h // merge_size, + merge_size, + w // merge_size, + merge_size, + hidden_size, + ) + media_embeds = media_embeds.permute(0, 1, 3, 2, 4, 5).contiguous() + merged_embeds.append( + media_embeds.view(-1, merge_size * merge_size * hidden_size) + ) + + if current_idx != image_embeds.shape[0]: + raise ValueError( + "image_embeds contains more patches than grid_thw describes: " + f"got {image_embeds.shape[0]}, expected {current_idx}." + ) + if not merged_embeds: + return image_embeds.new_empty((0, merge_size * merge_size * hidden_size)) + return torch.cat(merged_embeds, dim=0) + + +class Cosmos3EdgePatchMerger(nn.Module): + """ + Projector: LayerNorm -> Linear -> GELU -> Linear + + Reads config from projector_config (not vision_config). + input_hidden_size * spatial_merge_size² -> merger_intermediate_size + -> out_hidden_size + """ + + def __init__( + self, + input_hidden_size: int, + out_hidden_size: int, + merger_intermediate_size: int, + spatial_merge_size: int = 2, + use_postshuffle_norm: bool = False, + quant_config=None, + prefix: str = "", + ) -> None: + super().__init__() + use_data_parallel = is_vit_use_data_parallel() + self.spatial_merge_size = spatial_merge_size + self.hidden_size = input_hidden_size * (spatial_merge_size**2) + self.input_hidden_size = input_hidden_size + self.out_hidden_size = out_hidden_size + self.use_postshuffle_norm = use_postshuffle_norm + + norm_dim = self.hidden_size if use_postshuffle_norm else input_hidden_size + self.norm = nn.LayerNorm(norm_dim, eps=1e-6) + + self.linear_fc1 = ColumnParallelLinear( + self.hidden_size, + merger_intermediate_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.linear_fc1", + disable_tp=use_data_parallel, + ) + self.act_fn = nn.GELU() + self.linear_fc2 = RowParallelLinear( + merger_intermediate_size, + out_hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.linear_fc2", + disable_tp=use_data_parallel, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.use_postshuffle_norm: + x = self.norm(x.view(-1, self.hidden_size)) + else: + x = self.norm(x).view(-1, self.hidden_size) + + x, _ = self.linear_fc1(x) + x = self.act_fn(x) + x, _ = self.linear_fc2(x) + return x + + +class Cosmos3EdgeVisionModel(nn.Module): + """Complete Cosmos vision tower returning language-model embeddings.""" + + def __init__( + self, + vision_config, + projector_config, + quant_config=None, + prefix: str = "", + ) -> None: + super().__init__() + self.spatial_merge_size = projector_config.spatial_merge_size + self.out_hidden_size = projector_config.out_hidden_size + + self.encoder = Cosmos3EdgeVisionEncoder( + vision_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "encoder"), + ) + self.projector = Cosmos3EdgePatchMerger( + input_hidden_size=projector_config.input_hidden_size, + out_hidden_size=projector_config.out_hidden_size, + merger_intermediate_size=projector_config.merger_intermediate_size, + spatial_merge_size=self.spatial_merge_size, + use_postshuffle_norm=getattr( + projector_config, "use_postshuffle_norm", False + ), + quant_config=quant_config, + prefix=maybe_prefix(prefix, "projector"), + ) + + @property + def dtype(self) -> torch.dtype: + return self.encoder.dtype + + def forward( + self, + pixel_values: torch.Tensor, + grid_thw: torch.Tensor | list[list[int]], + ) -> torch.Tensor: + grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu") + image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw) + image_embeds = patch_merging_by_param( + image_embeds, + grid_thw, + merge_size=self.spatial_merge_size, + ) + image_embeds = image_embeds.view( + -1, + self.spatial_merge_size**2, + self.projector.input_hidden_size, + ) + return self.projector(image_embeds) + + +class Cosmos3EdgeProcessingInfo(Qwen3VLProcessingInfo): + def get_hf_config(self): + return self.ctx.get_hf_config() + + def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: + return self.ctx.get_hf_processor(**kwargs) + + +class Cosmos3EdgeMultiModalProcessor(Qwen3VLMultiModalProcessor): + pass + + +class Cosmos3EdgeDummyInputsBuilder(Qwen3VLDummyInputsBuilder): + pass + + +class Cosmos3EdgeAttention(NemotronHAttention): + """Nemotron-H attention with interleaved multimodal RoPE.""" + + def __init__( + self, + config: NemotronHConfig, + layer_idx: int, + model_config=None, + cache_config=None, + quant_config=None, + prefix: str = "", + ) -> None: + super().__init__( + config, + layer_idx, + model_config, + cache_config, + quant_config, + prefix, + ) + self.rotary_emb = get_rope( + self.head_dim, + max_position=config.max_position_embeddings, + rope_parameters=config.rope_parameters, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + **kwargs, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class Cosmos3EdgeAttentionDecoderLayer(nn.Module): + """Pre-norm attention layer for the Cosmos3 Edge dense text model.""" + + def __init__( + self, + config: NemotronHConfig, + layer_idx: int, + model_config=None, + cache_config=None, + quant_config=None, + parallel_config=None, + prefix: str = "", + ) -> None: + super().__init__() + get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None) + layer_config = get_layer_config(layer_idx) if get_layer_config else config + self.mixer = Cosmos3EdgeAttention( + layer_config, + layer_idx, + model_config, + cache_config, + quant_config, + prefix=f"{prefix}.mixer", + ) + self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.norm(hidden_states) + else: + hidden_states, residual = self.norm(hidden_states, residual) + hidden_states = self.mixer(positions=positions, hidden_states=hidden_states) + return hidden_states, residual + + +class Cosmos3EdgeTextModel(NemotronHModel): + """Nemotron-H backbone with Cosmos-owned checkpoint name mapping.""" + + # The enclosing Cosmos model maps and packs checkpoint weights. Disabling + # Nemotron-H's checkpoint-specific hook prevents those names from being + # remapped a second time when AutoWeightsLoader descends into this module. + load_weights = None # type: ignore[assignment] + + +class Cosmos3EdgeForCausalLM(nn.Module): + """Minimal CausalLM wrapper for the Cosmos3 Edge dense text model.""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + hf_config = vllm_config.model_config.hf_config + text_config = getattr(hf_config, "text_config", hf_config) + vllm_config = vllm_config.with_hf_config(text_config) + + self.vllm_config = vllm_config + self.model_config = vllm_config.model_config + self.quant_config = vllm_config.quant_config + self.config = text_config + pattern = text_config.hybrid_override_pattern + unsupported = set(pattern) - {"*", "-"} + if unsupported: + raise ValueError( + "Cosmos3 Edge only supports attention (`*`) and MLP (`-`) " + f"layers, but found: {sorted(unsupported)}" + ) + if len(pattern) != text_config.num_hidden_layers: + raise ValueError( + "hybrid_override_pattern must contain one entry per layer: " + f"expected {text_config.num_hidden_layers}, " + f"got {len(pattern)}" + ) + + self.model = Cosmos3EdgeTextModel( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model"), + decoder_layer_types={ + "*": Cosmos3EdgeAttentionDecoderLayer, + "-": NemotronHMLPDecoderLayer, + }, + ) + self.lm_head = ParallelLMHead( + text_config.vocab_size, + text_config.hidden_size, + quant_config=self.quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = LogitsProcessor(text_config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ): + return self.model( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + +def _cosmos3_edge_diffusers_prefix_map() -> dict[str, str]: + """Map Diffusers blocks to interleaved Nemotron-H layers. + + The checkpoint stores attention and MLP in one ``layers.N`` block, while + the Nemotron-H model represents them as separate decoder layers. Therefore, + checkpoint block ``N`` maps its attention to layer ``2N`` and its MLP to + layer ``2N + 1``. + """ + mappings = { + "embed_tokens.": "language_model.model.embed_tokens.", + "norm.": "language_model.model.norm_f.", + } + for physical_idx in range(28): + attention_idx = 2 * physical_idx + mlp_idx = attention_idx + 1 + source_prefix = f"layers.{physical_idx}" + attention_prefix = f"language_model.model.layers.{attention_idx}" + mlp_prefix = f"language_model.model.layers.{mlp_idx}" + mappings.update( + { + f"{source_prefix}.input_layernorm.": f"{attention_prefix}.norm.", + f"{source_prefix}.self_attn.qkv_proj.": ( + f"{attention_prefix}.mixer.qkv_proj." + ), + f"{source_prefix}.self_attn.o_proj.": ( + f"{attention_prefix}.mixer.o_proj." + ), + f"{source_prefix}.post_attention_layernorm.": (f"{mlp_prefix}.norm."), + f"{source_prefix}.mlp.": f"{mlp_prefix}.mixer.", + } + ) + return mappings + + +@MULTIMODAL_REGISTRY.register_processor( + Cosmos3EdgeMultiModalProcessor, + info=Cosmos3EdgeProcessingInfo, + dummy_inputs=Cosmos3EdgeDummyInputsBuilder, +) +class Cosmos3EdgeForConditionalGeneration( + nn.Module, + SupportsMultiModal, + SupportsPP, + SupportsMRoPE, +): + """ + Cosmos3 Edge model with a SigLIP2 vision encoder. + + Architecture: + - self.visual: SigLIP2 encoder + patch merger + projector + - self.language_model: Cosmos3EdgeForCausalLM (pure attention + RoPE) + """ + + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".self_attn.q_proj": (".self_attn.qkv_proj", "q"), + ".self_attn.k_proj": (".self_attn.qkv_proj", "k"), + ".self_attn.v_proj": (".self_attn.qkv_proj", "v"), + }, + orig_to_new_prefix={ + **_cosmos3_edge_diffusers_prefix_map(), + "proj_in.": None, + "proj_out.": None, + "time_embedder.": None, + "action_proj_in.": None, + "action_proj_out.": None, + "audio_modality_embed": None, + "action_modality_embed": None, + "model.visual.": "visual.encoder.", + "model.projector.": "visual.projector.", + "lm_head.": "language_model.lm_head.", + "model.language_model.": "language_model.model.", + }, + orig_to_new_substr={ + "_moe_gen": None, + ".add_q_proj.": None, + ".add_k_proj.": None, + ".add_v_proj.": None, + ".to_add_out.": None, + ".norm_added_q.": None, + ".norm_added_k.": None, + ".self_attn.to_q.": ".self_attn.q_proj.", + ".self_attn.to_k.": ".self_attn.k_proj.", + ".self_attn.to_v.": ".self_attn.v_proj.", + ".self_attn.to_out.": ".self_attn.o_proj.", + "language_model.embeddings": "language_model.embed_tokens", + }, + ) + + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + } + + # Cosmos3 Edge unified Diffusers checkpoints store reasoner weights across + # transformer/ and vision_encoder/. Match both while excluding VAE weights. + allow_patterns_overrides = ["[tv]*er/*.safetensors"] + + supports_encoder_tp_data = True + + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality.startswith("image"): + return "<|vision_start|><|image_pad|><|vision_end|>" + if modality.startswith("video"): + return "<|vision_start|><|video_pad|><|vision_end|>" + raise ValueError(f"Unsupported modality: {modality}") + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"): + super().__init__() + + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + + self.config = config + self.multimodal_config = multimodal_config + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" + + with self._mark_tower_model(vllm_config, {"image", "video"}): + self.visual = Cosmos3EdgeVisionModel( + vision_config=config.vision_config, + projector_config=config.projector_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "visual"), + ) + + with self._mark_language_model(vllm_config): + self.language_model = Cosmos3EdgeForCausalLM( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors + ) + + def _get_image_features( + self, + pixel_values: torch.Tensor, + grid_thw: torch.Tensor, + ) -> tuple[torch.Tensor, ...]: + """Run the complete vision tower and split by media item.""" + image_embeds = self.visual(pixel_values, grid_thw=grid_thw) + sizes = (grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist() + return image_embeds.split(sizes) + + def _parse_and_validate_image_input( + self, **kwargs: object + ) -> Qwen2_5_VLImageInputs | None: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + image_grid_thw = kwargs.pop("image_grid_thw", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + return Qwen2_5_VLImagePixelInputs( + type="pixel_values", + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + ) + + if image_embeds is not None: + return Qwen2_5_VLImageEmbeddingInputs( + type="image_embeds", + image_embeds=image_embeds, + image_grid_thw=image_grid_thw, + ) + + def _parse_and_validate_video_input( + self, **kwargs: object + ) -> Qwen2_5_VLVideoInputs | None: + pixel_values_videos = kwargs.pop("pixel_values_videos", None) + video_embeds = kwargs.pop("video_embeds", None) + video_grid_thw = kwargs.pop("video_grid_thw", None) + second_per_grid_ts = kwargs.pop("second_per_grid_ts", None) + + if pixel_values_videos is None and video_embeds is None: + return None + + if pixel_values_videos is not None: + return Qwen2_5_VLVideoPixelInputs( + type="pixel_values_videos", + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + second_per_grid_ts=second_per_grid_ts, + ) + + if video_embeds is not None: + return Qwen2_5_VLVideoEmbeddingInputs( + type="video_embeds", + video_embeds=video_embeds, + video_grid_thw=video_grid_thw, + ) + + def _process_image_input( + self, image_input: Qwen2_5_VLImageInputs + ) -> tuple[torch.Tensor, ...]: + grid_thw = image_input["image_grid_thw"] + assert grid_thw.ndim == 2 + + if image_input["type"] == "image_embeds": + image_embeds = image_input["image_embeds"].type(self.visual.dtype) + merge_size = self.visual.spatial_merge_size + sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist() + return image_embeds.split(sizes) + + pixel_values = image_input["pixel_values"].type(self.visual.dtype) + if self.use_data_parallel: + return run_dp_sharded_mrope_vision_model( + self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d" + ) + return self._get_image_features(pixel_values, grid_thw) + + def _process_video_input( + self, video_input: Qwen2_5_VLVideoInputs + ) -> tuple[torch.Tensor, ...]: + grid_thw = video_input["video_grid_thw"] + assert grid_thw.ndim == 2 + + if video_input["type"] == "video_embeds": + video_embeds = video_input["video_embeds"].type(self.visual.dtype) + merge_size = self.visual.spatial_merge_size + sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist() + return video_embeds.split(sizes) + + pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype) + if self.use_data_parallel: + return run_dp_sharded_mrope_vision_model( + self.visual, pixel_values_videos, grid_thw.tolist(), rope_type="rope_3d" + ) + return self._get_image_features(pixel_values_videos, grid_thw) + + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: + modalities = {} + image_input = self._parse_and_validate_image_input(**kwargs) + if image_input is not None: + modalities["image"] = image_input + video_input = self._parse_and_validate_video_input(**kwargs) + if video_input is not None: + modalities["video"] = video_input + return modalities + + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None: + mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) + if not mm_input_by_modality: + return None + + multimodal_embeddings: tuple[torch.Tensor, ...] = () + + for modality in mm_input_by_modality: + multimodal_input = mm_input_by_modality[modality] + if modality == "image": + image_embeddings = self._process_image_input(multimodal_input) + multimodal_embeddings += tuple(image_embeddings) + if modality == "video": + video_embeddings = self._process_video_input(multimodal_input) + multimodal_embeddings += tuple(video_embeddings) + + return multimodal_embeddings + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + if intermediate_tensors is not None: + inputs_embeds = None + + return self.language_model.model( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + return self.language_model.compute_logits(hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + def get_mm_mapping(self) -> MultiModelKeys: + return MultiModelKeys.from_string_field( + language_model="language_model", + connector="visual.projector", + tower_model="visual.", + ) + + def get_mrope_input_positions( + self, + input_tokens: list[int], + mm_features: list[MultiModalFeatureSpec], + ) -> tuple[torch.Tensor, int]: + return Qwen3VLForConditionalGeneration._get_mrope_input_positions( + input_tokens=input_tokens, + mm_features=mm_features, + config=self.config, + ) diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index bd5cd358d8..016bff18ec 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -19,7 +19,7 @@ """Inference-only NemotronH model.""" import typing -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from itertools import islice import torch @@ -540,9 +540,22 @@ ALL_DECODER_LAYER_TYPES = { } -@support_torch_compile +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + } +) class NemotronHModel(nn.Module, EagleModelMixin): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + decoder_layer_types: Mapping[str, type[nn.Module]] | None = None, + ): super().__init__() config: NemotronHConfig = vllm_config.model_config.hf_config @@ -562,11 +575,16 @@ class NemotronHModel(nn.Module, EagleModelMixin): self.has_moe = "E" in config.hybrid_override_pattern + layer_types = decoder_layer_types or ALL_DECODER_LAYER_TYPES + def get_layer(prefix: str): layer_idx = int(prefix.rsplit(".", 1)[1]) - layer_class = ALL_DECODER_LAYER_TYPES[ - config.hybrid_override_pattern[layer_idx] - ] + layer_type = config.hybrid_override_pattern[layer_idx] + if layer_type not in layer_types: + raise ValueError( + f"Unsupported layer type {layer_type!r} at layer {layer_idx}" + ) + layer_class = layer_types[layer_type] return layer_class( config=config, layer_idx=layer_idx, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 807c0dc40b..c8486dac5f 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -349,6 +349,10 @@ _MULTIMODAL_MODELS = { "Cohere2VisionForConditionalGeneration", ), "Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"), + "Cosmos3EdgeForConditionalGeneration": ( + "cosmos3_edge", + "Cosmos3EdgeForConditionalGeneration", + ), "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"), "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"), "DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"), diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index a848bbe142..c0f3396b0b 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -81,6 +81,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( ops_colqwen3="OpsColQwen3Config", qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig", cosmos3_omni="Cosmos3Config", + cosmos3_edge="Cosmos3EdgeConfig", diffusion_gemma="DiffusionGemmaConfig", deepseek_vl_v2="DeepseekVLV2Config", deepseek_v32="DeepseekV3Config", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index f48b0dd6df..b64310507b 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -27,6 +27,10 @@ _CLASS_TO_MODULE: dict[str, str] = { "OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3", "Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3", "Cosmos3Config": "vllm.transformers_utils.configs.cosmos3", + "Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge", + "Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge", + "Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge", + "Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge", "DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma", "DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma", "DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2", @@ -107,6 +111,10 @@ __all__ = [ "OpsColQwen3Config", "Qwen3VLNemotronEmbedConfig", "Cosmos3Config", + "Cosmos3EdgeConfig", + "Cosmos3EdgeProjectorConfig", + "Cosmos3EdgeTextConfig", + "Cosmos3EdgeVisionConfig", "DiffusionGemmaConfig", "DiffusionGemmaTextConfig", "DeepseekVLV2Config", diff --git a/vllm/transformers_utils/configs/cosmos3_edge.py b/vllm/transformers_utils/configs/cosmos3_edge.py new file mode 100644 index 0000000000..743e7d5deb --- /dev/null +++ b/vllm/transformers_utils/configs/cosmos3_edge.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from transformers.configuration_utils import PretrainedConfig +from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig + +from .nemotron_h import NemotronHConfig + + +def _normalize_mrope_parameters(config: NemotronHConfig) -> None: + """Normalize the checkpoint's mRoPE configuration for vLLM.""" + rope_parameters = dict(getattr(config, "rope_parameters", None) or {}) + mrope_section = rope_parameters.get("mrope_section") + if mrope_section is None: + return + + rope_parameters["mrope_section"] = list(mrope_section) + rope_parameters.setdefault("mrope_interleaved", True) + config.rope_parameters = rope_parameters + + +class Cosmos3EdgeTextConfig(NemotronHConfig): + """Dense Cosmos3 Edge text config backed by Nemotron-H layers.""" + + model_type = "cosmos3_edge_text" + has_no_defaults_at_init = True + ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} + + def __init__( + self, + num_hidden_layers: int = 28, + hidden_act: str = "relu2", + rms_norm_eps: float = 1e-5, + **kwargs, + ) -> None: + super().__init__( + num_hidden_layers=2 * num_hidden_layers, + hybrid_override_pattern="*-" * num_hidden_layers, + mlp_hidden_act=hidden_act, + layer_norm_epsilon=rms_norm_eps, + **kwargs, + ) + + def to_dict(self) -> dict: + config_dict = super().to_dict() + config_dict["num_hidden_layers"] = self.num_hidden_layers // 2 + config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act") + config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon") + config_dict.pop("hybrid_override_pattern", None) + return config_dict + + +class Cosmos3EdgeProjectorConfig(PretrainedConfig): + model_type = "cosmos3_edge_projector" + + def __init__( + self, + input_hidden_size: int = 1024, + use_postshuffle_norm: bool = False, + spatial_merge_size: int = 2, + merger_intermediate_size: int = 11520, + out_hidden_size: int = 1024, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.input_hidden_size = input_hidden_size + self.use_postshuffle_norm = use_postshuffle_norm + self.spatial_merge_size = spatial_merge_size + self.merger_intermediate_size = merger_intermediate_size + self.out_hidden_size = out_hidden_size + + +class Cosmos3EdgeVisionConfig(Siglip2VisionConfig): + model_type = "cosmos3_edge_vision" + + +class Cosmos3EdgeConfig(PretrainedConfig): + model_type = "cosmos3_edge" + keys_to_ignore_at_inference = ["past_key_values"] + has_no_defaults_at_init = True + sub_configs = { + "vision_config": Cosmos3EdgeVisionConfig, + "projector_config": Cosmos3EdgeProjectorConfig, + "text_config": Cosmos3EdgeTextConfig, + } + + def __init__( + self, + text_config: Cosmos3EdgeTextConfig | dict | None = None, + vision_config: Cosmos3EdgeVisionConfig | dict | None = None, + projector_config: Cosmos3EdgeProjectorConfig | dict | None = None, + image_token_id: int = 19, + video_token_id: int = 18, + vision_start_token_id: int = 20, + vision_end_token_id: int = 21, + tie_word_embeddings: bool = False, + **kwargs, + ) -> None: + if text_config is None: + text_config = {} + if vision_config is None: + vision_config = {} + if projector_config is None: + projector_config = {} + + self.text_config = ( + Cosmos3EdgeTextConfig(**text_config) + if isinstance(text_config, dict) + else text_config + ) + _normalize_mrope_parameters(self.text_config) + + self.vision_config = ( + Cosmos3EdgeVisionConfig(**vision_config) + if isinstance(vision_config, dict) + else vision_config + ) + self.projector_config = ( + Cosmos3EdgeProjectorConfig(**projector_config) + if isinstance(projector_config, dict) + else projector_config + ) + + # Qwen3-VL processing and M-RoPE read these attributes from the + # vision config, while this checkpoint stores them in projector config. + self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size + self.vision_config.temporal_patch_size = 1 + self.vision_config.out_hidden_size = self.projector_config.out_hidden_size + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.vision_end_token_id = vision_end_token_id + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +__all__ = [ + "Cosmos3EdgeConfig", + "Cosmos3EdgeProjectorConfig", + "Cosmos3EdgeTextConfig", + "Cosmos3EdgeVisionConfig", +] diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index 60eb2922fd..e35218c4be 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -14,6 +14,7 @@ __all__ = [ "BagelProcessor", "CheersProcessor", "CohereASRProcessor", + "Cosmos3EdgeProcessor", "DeepseekVLV2Processor", "FireRedASR2Processor", "FireRedLIDProcessor", @@ -49,6 +50,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "BagelProcessor": "vllm.transformers_utils.processors.bagel", "CheersProcessor": "vllm.transformers_utils.processors.cheers", "CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr", + "Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge", "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2", "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2", "FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid", diff --git a/vllm/transformers_utils/processors/cosmos3_edge.py b/vllm/transformers_utils/processors/cosmos3_edge.py new file mode 100644 index 0000000000..2644c6b17e --- /dev/null +++ b/vllm/transformers_utils/processors/cosmos3_edge.py @@ -0,0 +1,356 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +import math + +import torch +from torchvision.transforms import InterpolationMode +from transformers import AutoTokenizer +from transformers.feature_extraction_utils import BatchFeature +from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict +from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor +from transformers.models.qwen3_vl.video_processing_qwen3_vl import ( + Qwen3VLVideoProcessor, + Qwen3VLVideoProcessorInitKwargs, + get_image_size, + smart_resize, +) +from transformers.models.siglip2.image_processing_siglip2 import ( + Siglip2ImageProcessor, + convert_image_to_patches, +) +from transformers.processing_utils import Unpack +from transformers.utils import TensorType +from transformers.video_utils import group_videos_by_shape, reorder_videos + + +def round_by_factor(number: int, factor: int) -> int: + """Returns the closest integer to 'number' that is divisible by 'factor'.""" + return round(number / factor) * factor + + +def ceil_by_factor(number: int, factor: int) -> int: + """Return the smallest multiple of ``factor`` greater than ``number``.""" + return math.ceil(number / factor) * factor + + +def floor_by_factor(number: int, factor: int) -> int: + """Return the largest multiple of ``factor`` less than ``number``.""" + return math.floor(number / factor) * factor + + +class Cosmos3EdgeImagesKwargs( # type: ignore[call-arg] + Siglip2ImageProcessor.valid_kwargs, + total=False, +): + # global setting for all images, can be overridden by per-image kwargs + max_pixels: int | None + min_pixels: int | None + + per_image_kwargs: list[dict] | None + + +class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor): + resample = PILImageResampling.BICUBIC + valid_kwargs = Cosmos3EdgeImagesKwargs + + def __init__(self, **kwargs: Unpack[Cosmos3EdgeImagesKwargs]): + super().__init__(**kwargs) + + def _resize_image( + self, + image: torch.Tensor, + max_ratio=200, + resample: PILImageResampling | InterpolationMode | int | None = None, + **kwargs, + ) -> torch.Tensor: + """ + Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + 3. The aspect ratio of the image is maintained as closely as possible. + """ + image_min_pixels = kwargs.get("min_pixels") or self.size.get( + "shortest_edge", None + ) + image_max_pixels = kwargs.get("max_pixels") or self.size.get( + "longest_edge", None + ) + assert image_min_pixels is not None and image_max_pixels is not None, ( + "When do_resize is True, min_pixels and max_pixels must be provided." + ) + assert image_max_pixels >= image_min_pixels, ( + "The max_pixels of image must be greater than or equal to min_pixels." + ) + + _, height, width = image.shape + aspect_ratio = max(height, width) / min(height, width) + if aspect_ratio > max_ratio: + raise ValueError( + f"absolute aspect ratio must be smaller than {max_ratio}, " + f"got {aspect_ratio}" + ) + factor = self.merge_size * self.patch_size + h_bar = max(factor, round_by_factor(height, factor)) + w_bar = max(factor, round_by_factor(width, factor)) + if h_bar * w_bar > image_max_pixels: + beta = math.sqrt((height * width) / image_max_pixels) + h_bar = floor_by_factor(height / beta, factor) + w_bar = floor_by_factor(width / beta, factor) + elif h_bar * w_bar < image_min_pixels: + beta = math.sqrt(image_min_pixels / (height * width)) + h_bar = ceil_by_factor(height * beta, factor) + w_bar = ceil_by_factor(width * beta, factor) + + image = self.resize( + image, + size=SizeDict(height=h_bar, width=w_bar), + resample=resample, + ) + return image + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + patch_size: int, + max_num_patches: int, + resample: PILImageResampling | InterpolationMode | int | None, + do_rescale: bool, + rescale_factor: float, + do_normalize: bool, + image_mean: float | list[float] | None, + image_std: float | list[float] | None, + return_tensors: str | TensorType | None, + **kwargs, + ) -> BatchFeature: + per_image_kwargs = kwargs.pop("per_image_kwargs", None) + pixel_values = [] + spatial_shapes = [] + for idx, image in enumerate(images): + # merge global kwargs with per-image overrides (if provided) + image_kwargs = kwargs.copy() + if ( + per_image_kwargs is not None + and idx < len(per_image_kwargs) + and per_image_kwargs[idx] + ): + image_kwargs.update(per_image_kwargs[idx]) + if do_resize: + image = self._resize_image( + image, + max_ratio=200, + resample=resample, + **image_kwargs, + ) + + image = self.rescale_and_normalize( + image, do_rescale, rescale_factor, do_normalize, image_mean, image_std + ) + + # (channels, height, width) -> (patches, patch_size**2 * channels) + patches = convert_image_to_patches(image, patch_size) + + num_patches_height = image.shape[1] // patch_size + num_patches_width = image.shape[2] // patch_size + + spatial_shapes.append((num_patches_height, num_patches_width)) + pixel_values.append(patches) + + pixel_values = torch.cat(pixel_values, dim=0) + + spatial_shapes = torch.tensor(spatial_shapes) + t_dim = torch.ones( + (spatial_shapes.shape[0], 1), + dtype=spatial_shapes.dtype, + device=spatial_shapes.device, + ) + image_grid_thw = torch.cat([t_dim, spatial_shapes], dim=1) + + batch_feature = BatchFeature( + data={ + "pixel_values": pixel_values, + "image_grid_thw": image_grid_thw, + }, + tensor_type=return_tensors, + ) + return batch_feature + + +class Cosmos3EdgeVideoProcessor(Qwen3VLVideoProcessor): + def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]): + super().__init__(**kwargs) + + def _preprocess( + self, + videos: list[torch.Tensor], + do_convert_rgb: bool = True, + do_resize: bool = True, + size: SizeDict | None = None, + resample: PILImageResampling + | InterpolationMode + | int + | None = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: float = 1 / 255.0, + do_normalize: bool = True, + image_mean: float | list[float] | None = None, + image_std: float | list[float] | None = None, + patch_size: int | None = None, + return_tensors: str | TensorType | None = None, + **kwargs, + ): + assert patch_size is not None + + merge_size = self.merge_size + grouped_videos, grouped_videos_index = group_videos_by_shape(videos) + resized_videos_grouped = {} + + for shape, stacked_videos in grouped_videos.items(): + B, T, C, H, W = stacked_videos.shape + num_frames, height, width = T, H, W + if do_resize: + assert size is not None + # 1. Both dimensions (height and width) are divisible by 'factor'. + # 2. Total pixels are within the configured min/max range. + resized_height, resized_width = smart_resize( + num_frames=num_frames, + height=height, + width=width, + temporal_factor=1, + factor=patch_size * merge_size, + min_pixels=size.shortest_edge, + max_pixels=size.longest_edge, + ) + stacked_videos = stacked_videos.view(B * T, C, H, W) + stacked_videos = self.resize( + stacked_videos, + size=SizeDict(height=resized_height, width=resized_width), + resample=resample, + ) + stacked_videos = stacked_videos.view( + B, T, C, resized_height, resized_width + ) + resized_videos_grouped[shape] = stacked_videos + resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) + + # Group videos by size for further processing + # Resize may return videos with different sizes. + grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) + processed_videos_grouped = {} + processed_grids = {} + for shape, stacked_videos in grouped_videos.items(): + resized_height, resized_width = get_image_size( + stacked_videos[0], channel_dim=ChannelDimension.FIRST + ) + # Fused rescale and normalize + stacked_videos = self.rescale_and_normalize( + stacked_videos, + do_rescale, + rescale_factor, + do_normalize, + image_mean, + image_std, + ) + patches = stacked_videos + + batch_size, grid_t, channel = patches.shape[:3] + grid_h, grid_w = resized_height // patch_size, resized_width // patch_size + + # [1, 16, 3, 320, 608] -> [1, 16, 3, 20, 16, 38, 16] + patches = patches.view( + batch_size, # 0 + grid_t, # 1 + channel, # 2 + grid_h, # 3 + patch_size, # 4 + grid_w, # 5 + patch_size, # 6 + ) + # [1, 16, 3, 20, 16, 38, 16] -> [1, 16, 20, 38, 16, 16, 3] + # Result: [batch, grid_t, grid_h, grid_w, patch_h, patch_w, channel] + patches = patches.permute(0, 1, 3, 5, 4, 6, 2) + + # [1, 16, 20, 38, 16, 16, 3] -> [1, 5120, 768] + flatten_patches = patches.reshape( + batch_size, + grid_t * grid_h * grid_w, + patch_size * patch_size * channel, + ) + + processed_videos_grouped[shape] = flatten_patches + processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size + + processed_videos = reorder_videos( + processed_videos_grouped, grouped_videos_index + ) + processed_grids = reorder_videos(processed_grids, grouped_videos_index) + pixel_values_videos = torch.cat(processed_videos, dim=0) + video_grid_thw = torch.tensor(processed_grids) + data = { + "pixel_values_videos": pixel_values_videos, + "video_grid_thw": video_grid_thw, + } + + return BatchFeature(data=data, tensor_type=return_tensors) + + +class Cosmos3EdgeProcessor(Qwen3VLProcessor): + """Construct a processor for Cosmos3 Edge multimodal inputs. + + This processor combines :class:`Cosmos3EdgeImageProcessor`, + :class:`Cosmos3EdgeVideoProcessor`, and a tokenizer into a single + interface. It inherits the text, image, and video input handling and the + decoding methods from :class:`Qwen3VLProcessor`, while using the Cosmos3 + Edge-specific image and video preprocessing implementations. + + Args: + image_processor: A :class:`Cosmos3EdgeImageProcessor` used to resize, + normalize, and patch image inputs. + tokenizer: The tokenizer used to encode and decode text. + video_processor: A :class:`Cosmos3EdgeVideoProcessor` used to resize, + normalize, and patch video inputs. + chat_template: An optional Jinja template used to render chat messages + into a tokenizable string. + """ + + attributes = ["image_processor", "tokenizer", "video_processor"] + + # Transformers' auto loaders cannot resolve these vLLM-native image and + # video processors without remote code or global auto-class registration. + @classmethod + def _get_arguments_from_pretrained( + cls, + pretrained_model_name_or_path, + processor_dict=None, + **kwargs, + ): + component_kwargs = dict(kwargs) + component_kwargs.pop("trust_remote_code", None) + + image_processor = Cosmos3EdgeImageProcessor.from_pretrained( + pretrained_model_name_or_path, + **component_kwargs, + ) + tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, + trust_remote_code=False, + **component_kwargs, + ) + video_processor = Cosmos3EdgeVideoProcessor.from_pretrained( + pretrained_model_name_or_path, + **component_kwargs, + ) + + return [ + image_processor, + tokenizer, + video_processor, + ] + + +__all__ = [ + "Cosmos3EdgeProcessor", +]