diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index f19f01c698..37d27234aa 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -514,6 +514,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ |
| `Cohere2VisionForConditionalGeneration` | Command A Vision, Command-A+ | T + I+ | `CohereLabs/command-a-vision-07-2025`, `CohereLabs/command-a-plus-05-2026`, etc. | | ✅︎ |
| `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | | ✅︎ |
+| `Cosmos3EdgeForConditionalGeneration` | Cosmos3-Edge (understanding tower) | T + IE+ + VE+ | `nvidia/Cosmos3-Edge` | | ✅︎ |
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I+ | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I+ | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I+ | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
diff --git a/tests/models/multimodal/processing/test_cosmos3_edge.py b/tests/models/multimodal/processing/test_cosmos3_edge.py
new file mode 100644
index 0000000000..5ebef08c10
--- /dev/null
+++ b/tests/models/multimodal/processing/test_cosmos3_edge.py
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+
+import pytest
+
+from vllm.assets.video import VideoAsset
+from vllm.config import ModelConfig
+from vllm.multimodal import MULTIMODAL_REGISTRY
+
+from ....conftest import ImageTestAssets
+from ...utils import build_model_context
+
+MODEL_ID = "nvidia/Cosmos3-Edge"
+IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
+VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
+LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
+
+
+@pytest.fixture(scope="module")
+def processor():
+ if LOCAL_MODEL_PATH:
+ model_config = ModelConfig(
+ LOCAL_MODEL_PATH,
+ tokenizer=LOCAL_MODEL_PATH,
+ max_model_len=4096,
+ limit_mm_per_prompt={"image": 2, "video": 1},
+ )
+ else:
+ ctx = build_model_context(
+ MODEL_ID,
+ limit_mm_per_prompt={"image": 2, "video": 1},
+ )
+ model_config = ctx.model_config
+
+ return MULTIMODAL_REGISTRY.create_processor(model_config)
+
+
+def _assert_image_outputs(processor, processed, num_images: int) -> None:
+ mm_data = processed["mm_kwargs"].get_data()
+ grid_thw = mm_data["image_grid_thw"]
+ pixel_values = mm_data["pixel_values"]
+
+ assert grid_thw.shape == (num_images, 3)
+ assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
+
+ merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
+ expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
+ image_placeholders = processed["mm_placeholders"]["image"]
+
+ assert len(image_placeholders) == num_images
+ assert [placeholder.length for placeholder in image_placeholders] == (
+ expected_tokens
+ )
+
+ image_token_id = processor.info.get_hf_processor().image_token_id
+ assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
+
+
+def _assert_video_outputs(processor, processed) -> None:
+ mm_data = processed["mm_kwargs"].get_data()
+ grid_thw = mm_data["video_grid_thw"]
+ pixel_values = mm_data["pixel_values_videos"]
+
+ assert grid_thw.shape == (1, 3)
+ assert pixel_values.shape[0] == int(grid_thw.prod())
+ assert len(processed["mm_placeholders"]["video"]) == 1
+
+ merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
+ expected_tokens = int(grid_thw.prod()) // merge_size**2
+ video_token_id = processor.info.get_hf_config().video_token_id
+ assert processed["prompt_token_ids"].count(video_token_id) == expected_tokens
+
+
+@pytest.mark.parametrize("num_images", [1, 2])
+def test_process_images(
+ processor,
+ image_assets: ImageTestAssets,
+ num_images: int,
+) -> None:
+ images = [asset.pil_image for asset in image_assets[:num_images]]
+ processed = processor(
+ IMAGE_PLACEHOLDER * num_images,
+ mm_items=processor.info.parse_mm_data({"image": images}),
+ hf_processor_mm_kwargs={},
+ )
+
+ _assert_image_outputs(processor, processed, num_images)
+
+
+def test_process_video(processor) -> None:
+ video_asset = VideoAsset(name="baby_reading", num_frames=8)
+ video = (video_asset.np_ndarrays, video_asset.metadata)
+ processed = processor(
+ VIDEO_PLACEHOLDER,
+ mm_items=processor.info.parse_mm_data({"video": [video]}),
+ hf_processor_mm_kwargs={},
+ )
+
+ _assert_video_outputs(processor, processed)
diff --git a/tests/models/multimodal/test_mapping.py b/tests/models/multimodal/test_mapping.py
index b1081726f2..ce9b8a9d45 100644
--- a/tests/models/multimodal/test_mapping.py
+++ b/tests/models/multimodal/test_mapping.py
@@ -74,6 +74,77 @@ def test_cosmos3_new_checkpoint_weights_mapper():
)
+def test_cosmos3_edge_checkpoint_weights_mapper():
+ from vllm.model_executor.models.cosmos3_edge import (
+ Cosmos3EdgeForConditionalGeneration,
+ )
+
+ mapper = Cosmos3EdgeForConditionalGeneration.hf_to_vllm_mapper
+
+ assert mapper.apply_list(
+ [
+ "embed_tokens.weight",
+ "norm.weight",
+ "layers.0.input_layernorm.weight",
+ "layers.0.self_attn.to_q.weight",
+ "layers.0.self_attn.to_k.weight",
+ "layers.0.self_attn.to_v.weight",
+ "layers.0.self_attn.to_out.weight",
+ "layers.0.post_attention_layernorm.weight",
+ "layers.0.mlp.up_proj.weight",
+ "layers.0.mlp.down_proj.weight",
+ "layers.27.input_layernorm.weight",
+ "layers.27.self_attn.to_q.weight",
+ "layers.27.post_attention_layernorm.weight",
+ "layers.27.mlp.down_proj.weight",
+ "model.visual.embeddings.patch_embedding.weight",
+ "model.visual.encoder.layers.0.self_attn.q_proj.weight",
+ "model.projector.linear_fc1.weight",
+ "lm_head.weight",
+ ]
+ ) == [
+ "language_model.model.embed_tokens.weight",
+ "language_model.model.norm_f.weight",
+ "language_model.model.layers.0.norm.weight",
+ "language_model.model.layers.0.mixer.qkv_proj.weight",
+ "language_model.model.layers.0.mixer.qkv_proj.weight",
+ "language_model.model.layers.0.mixer.qkv_proj.weight",
+ "language_model.model.layers.0.mixer.o_proj.weight",
+ "language_model.model.layers.1.norm.weight",
+ "language_model.model.layers.1.mixer.up_proj.weight",
+ "language_model.model.layers.1.mixer.down_proj.weight",
+ "language_model.model.layers.54.norm.weight",
+ "language_model.model.layers.54.mixer.qkv_proj.weight",
+ "language_model.model.layers.55.norm.weight",
+ "language_model.model.layers.55.mixer.down_proj.weight",
+ "visual.encoder.embeddings.patch_embedding.weight",
+ "visual.encoder.encoder.layers.0.self_attn.qkv_proj.weight",
+ "visual.projector.linear_fc1.weight",
+ "language_model.lm_head.weight",
+ ]
+
+ assert (
+ mapper.apply_list(
+ [
+ "layers.0.self_attn.add_q_proj.weight",
+ "layers.0.self_attn.add_k_proj.weight",
+ "layers.0.self_attn.add_v_proj.weight",
+ "layers.0.self_attn.to_add_out.weight",
+ "layers.0.self_attn.norm_added_q.weight",
+ "layers.0.self_attn.norm_added_k.weight",
+ "layers.0.self_attn.q_proj_moe_gen.weight",
+ "layers.0.mlp_moe_gen.up_proj.weight",
+ "norm_moe_gen.weight",
+ "proj_in.weight",
+ "proj_out.weight",
+ "audio_modality_embed",
+ "action_modality_embed",
+ ]
+ )
+ == []
+ )
+
+
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
metadata = try_get_safetensors_metadata(repo)
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 4157d23984..4e8fe96fdb 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -811,6 +811,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
max_model_len=4096,
min_transformers_version="4.57",
),
+ "Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
+ "nvidia/Cosmos3-Edge",
+ max_model_len=4096,
+ is_available_online=False,
+ ),
"DeepseekVLV2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/deepseek-vl2-tiny",
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
diff --git a/vllm/model_executor/models/cosmos3.py b/vllm/model_executor/models/cosmos3.py
index e0ba8a247d..71629196a7 100644
--- a/vllm/model_executor/models/cosmos3.py
+++ b/vllm/model_executor/models/cosmos3.py
@@ -3,8 +3,6 @@
import regex
-from vllm.config import VllmConfig
-from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm.model_executor.models.utils import WeightsMapper
@@ -52,22 +50,7 @@ class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
},
)
- allow_patterns_overrides = ["transformer/*.safetensors"]
-
- """
- Cosmos3 checkpoint separates transformer weights and vision_encoder weights
- into separate directories, as it's in diffusers checkpoint format.
- Using secondary_weights here to load all necessary weights for
- the Reasoner-only part.
- """
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
- super().__init__(vllm_config=vllm_config, prefix=prefix)
- self.secondary_weights = [
- DefaultModelLoader.Source(
- model_or_path=vllm_config.model_config.model,
- revision=vllm_config.model_config.revision,
- prefix="",
- allow_patterns_overrides=["vision_encoder/*.safetensors"],
- ),
- ]
+ # Cosmos3 unified Diffusers checkpoints store reasoner weights across
+ # transformer/ and vision_encoder/. Match both while excluding VAE and
+ # sound_tokenizer weights; the mapper drops generation-only tensors.
+ allow_patterns_overrides = ["[tv]*er/*.safetensors"]
diff --git a/vllm/model_executor/models/cosmos3_edge.py b/vllm/model_executor/models/cosmos3_edge.py
new file mode 100644
index 0000000000..be61d564cc
--- /dev/null
+++ b/vllm/model_executor/models/cosmos3_edge.py
@@ -0,0 +1,751 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Iterable
+
+import torch
+import torch.nn as nn
+from transformers import ProcessorMixin
+
+from vllm.config import VllmConfig
+from vllm.model_executor.layers.layernorm import RMSNorm
+from vllm.model_executor.layers.linear import (
+ ColumnParallelLinear,
+ RowParallelLinear,
+)
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
+from vllm.model_executor.layers.rotary_embedding import get_rope
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
+from vllm.model_executor.models.module_mapping import MultiModelKeys
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import MultiModalFeatureSpec
+from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
+
+from .interfaces import (
+ MultiModalEmbeddings,
+ SupportsMRoPE,
+ SupportsMultiModal,
+ SupportsPP,
+)
+from .lfm2_siglip2 import Siglip2VisionTransformer
+from .nemotron_h import (
+ NemotronHAttention,
+ NemotronHMLPDecoderLayer,
+ NemotronHModel,
+)
+from .qwen2_5_vl import (
+ Qwen2_5_VLImageEmbeddingInputs,
+ Qwen2_5_VLImageInputs,
+ Qwen2_5_VLImagePixelInputs,
+ Qwen2_5_VLVideoEmbeddingInputs,
+ Qwen2_5_VLVideoInputs,
+ Qwen2_5_VLVideoPixelInputs,
+)
+from .qwen3_vl import (
+ Qwen3VLDummyInputsBuilder,
+ Qwen3VLForConditionalGeneration,
+ Qwen3VLMultiModalProcessor,
+ Qwen3VLProcessingInfo,
+)
+from .utils import (
+ AutoWeightsLoader,
+ WeightsMapper,
+ maybe_prefix,
+)
+from .vision import (
+ is_vit_use_data_parallel,
+ run_dp_sharded_mrope_vision_model,
+)
+
+
+class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
+ """Adapts Cosmos (T, H, W) metadata to vLLM packed SigLIP2."""
+
+ @property
+ def dtype(self) -> torch.dtype:
+ return self.embeddings.patch_embedding.weight.dtype
+
+ def forward(
+ self,
+ pixel_values: torch.Tensor,
+ grid_thw: torch.Tensor,
+ ) -> torch.Tensor:
+ # SigLIP2 attends to each frame independently, so expand every THW
+ # entry into T separate HW attention sequences.
+ grid_thw_cpu = grid_thw.to(device="cpu")
+ spatial_shapes = torch.repeat_interleave(
+ grid_thw_cpu[:, 1:],
+ grid_thw_cpu[:, 0],
+ dim=0,
+ )
+ lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
+ lengths = lengths_cpu.to(
+ device=pixel_values.device,
+ non_blocking=True,
+ )
+
+ cu_seqlens = torch.zeros(
+ lengths.numel() + 1,
+ dtype=torch.int32,
+ device=pixel_values.device,
+ )
+ cu_seqlens[1:] = lengths.cumsum(dim=0)
+
+ max_seqlen = lengths_cpu.max().reshape(1)
+
+ return super().forward(
+ pixel_values_packed=pixel_values,
+ spatial_shapes=spatial_shapes,
+ cu_seqlens=cu_seqlens,
+ max_seqlen=max_seqlen,
+ )[0]
+
+
+def patch_merging_by_param(
+ image_embeds: torch.Tensor,
+ grid_thw: torch.Tensor,
+ merge_size: int = 2,
+) -> torch.Tensor:
+ """Merge each spatial ``merge_size`` block into the channel dimension."""
+ merged_embeds: list[torch.Tensor] = []
+ current_idx = 0
+ hidden_size = image_embeds.shape[-1]
+
+ for grid in grid_thw:
+ t, h, w = grid.tolist()
+ if h % merge_size != 0 or w % merge_size != 0:
+ raise ValueError(
+ "Grid height and width must be divisible by merge_size: "
+ f"got grid={(t, h, w)} and merge_size={merge_size}."
+ )
+
+ num_patches = t * h * w
+ media_embeds = image_embeds[current_idx : current_idx + num_patches]
+ if media_embeds.shape[0] != num_patches:
+ raise ValueError(
+ "image_embeds contains fewer patches than grid_thw describes."
+ )
+ current_idx += num_patches
+
+ media_embeds = media_embeds.view(t, h, w, hidden_size)
+ media_embeds = media_embeds.view(
+ t,
+ h // merge_size,
+ merge_size,
+ w // merge_size,
+ merge_size,
+ hidden_size,
+ )
+ media_embeds = media_embeds.permute(0, 1, 3, 2, 4, 5).contiguous()
+ merged_embeds.append(
+ media_embeds.view(-1, merge_size * merge_size * hidden_size)
+ )
+
+ if current_idx != image_embeds.shape[0]:
+ raise ValueError(
+ "image_embeds contains more patches than grid_thw describes: "
+ f"got {image_embeds.shape[0]}, expected {current_idx}."
+ )
+ if not merged_embeds:
+ return image_embeds.new_empty((0, merge_size * merge_size * hidden_size))
+ return torch.cat(merged_embeds, dim=0)
+
+
+class Cosmos3EdgePatchMerger(nn.Module):
+ """
+ Projector: LayerNorm -> Linear -> GELU -> Linear
+
+ Reads config from projector_config (not vision_config).
+ input_hidden_size * spatial_merge_size² -> merger_intermediate_size
+ -> out_hidden_size
+ """
+
+ def __init__(
+ self,
+ input_hidden_size: int,
+ out_hidden_size: int,
+ merger_intermediate_size: int,
+ spatial_merge_size: int = 2,
+ use_postshuffle_norm: bool = False,
+ quant_config=None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ use_data_parallel = is_vit_use_data_parallel()
+ self.spatial_merge_size = spatial_merge_size
+ self.hidden_size = input_hidden_size * (spatial_merge_size**2)
+ self.input_hidden_size = input_hidden_size
+ self.out_hidden_size = out_hidden_size
+ self.use_postshuffle_norm = use_postshuffle_norm
+
+ norm_dim = self.hidden_size if use_postshuffle_norm else input_hidden_size
+ self.norm = nn.LayerNorm(norm_dim, eps=1e-6)
+
+ self.linear_fc1 = ColumnParallelLinear(
+ self.hidden_size,
+ merger_intermediate_size,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.linear_fc1",
+ disable_tp=use_data_parallel,
+ )
+ self.act_fn = nn.GELU()
+ self.linear_fc2 = RowParallelLinear(
+ merger_intermediate_size,
+ out_hidden_size,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.linear_fc2",
+ disable_tp=use_data_parallel,
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ if self.use_postshuffle_norm:
+ x = self.norm(x.view(-1, self.hidden_size))
+ else:
+ x = self.norm(x).view(-1, self.hidden_size)
+
+ x, _ = self.linear_fc1(x)
+ x = self.act_fn(x)
+ x, _ = self.linear_fc2(x)
+ return x
+
+
+class Cosmos3EdgeVisionModel(nn.Module):
+ """Complete Cosmos vision tower returning language-model embeddings."""
+
+ def __init__(
+ self,
+ vision_config,
+ projector_config,
+ quant_config=None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ self.spatial_merge_size = projector_config.spatial_merge_size
+ self.out_hidden_size = projector_config.out_hidden_size
+
+ self.encoder = Cosmos3EdgeVisionEncoder(
+ vision_config,
+ quant_config=quant_config,
+ prefix=maybe_prefix(prefix, "encoder"),
+ )
+ self.projector = Cosmos3EdgePatchMerger(
+ input_hidden_size=projector_config.input_hidden_size,
+ out_hidden_size=projector_config.out_hidden_size,
+ merger_intermediate_size=projector_config.merger_intermediate_size,
+ spatial_merge_size=self.spatial_merge_size,
+ use_postshuffle_norm=getattr(
+ projector_config, "use_postshuffle_norm", False
+ ),
+ quant_config=quant_config,
+ prefix=maybe_prefix(prefix, "projector"),
+ )
+
+ @property
+ def dtype(self) -> torch.dtype:
+ return self.encoder.dtype
+
+ def forward(
+ self,
+ pixel_values: torch.Tensor,
+ grid_thw: torch.Tensor | list[list[int]],
+ ) -> torch.Tensor:
+ grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu")
+ image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw)
+ image_embeds = patch_merging_by_param(
+ image_embeds,
+ grid_thw,
+ merge_size=self.spatial_merge_size,
+ )
+ image_embeds = image_embeds.view(
+ -1,
+ self.spatial_merge_size**2,
+ self.projector.input_hidden_size,
+ )
+ return self.projector(image_embeds)
+
+
+class Cosmos3EdgeProcessingInfo(Qwen3VLProcessingInfo):
+ def get_hf_config(self):
+ return self.ctx.get_hf_config()
+
+ def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
+ return self.ctx.get_hf_processor(**kwargs)
+
+
+class Cosmos3EdgeMultiModalProcessor(Qwen3VLMultiModalProcessor):
+ pass
+
+
+class Cosmos3EdgeDummyInputsBuilder(Qwen3VLDummyInputsBuilder):
+ pass
+
+
+class Cosmos3EdgeAttention(NemotronHAttention):
+ """Nemotron-H attention with interleaved multimodal RoPE."""
+
+ def __init__(
+ self,
+ config: NemotronHConfig,
+ layer_idx: int,
+ model_config=None,
+ cache_config=None,
+ quant_config=None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__(
+ config,
+ layer_idx,
+ model_config,
+ cache_config,
+ quant_config,
+ prefix,
+ )
+ self.rotary_emb = get_rope(
+ self.head_dim,
+ max_position=config.max_position_embeddings,
+ rope_parameters=config.rope_parameters,
+ )
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ **kwargs,
+ ) -> torch.Tensor:
+ qkv, _ = self.qkv_proj(hidden_states)
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q, k = self.rotary_emb(positions, q, k)
+ attn_output = self.attn(q, k, v)
+ output, _ = self.o_proj(attn_output)
+ return output
+
+
+class Cosmos3EdgeAttentionDecoderLayer(nn.Module):
+ """Pre-norm attention layer for the Cosmos3 Edge dense text model."""
+
+ def __init__(
+ self,
+ config: NemotronHConfig,
+ layer_idx: int,
+ model_config=None,
+ cache_config=None,
+ quant_config=None,
+ parallel_config=None,
+ prefix: str = "",
+ ) -> None:
+ super().__init__()
+ get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
+ layer_config = get_layer_config(layer_idx) if get_layer_config else config
+ self.mixer = Cosmos3EdgeAttention(
+ layer_config,
+ layer_idx,
+ model_config,
+ cache_config,
+ quant_config,
+ prefix=f"{prefix}.mixer",
+ )
+ self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ residual: torch.Tensor | None,
+ **kwargs,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ if residual is None:
+ residual = hidden_states
+ hidden_states = self.norm(hidden_states)
+ else:
+ hidden_states, residual = self.norm(hidden_states, residual)
+ hidden_states = self.mixer(positions=positions, hidden_states=hidden_states)
+ return hidden_states, residual
+
+
+class Cosmos3EdgeTextModel(NemotronHModel):
+ """Nemotron-H backbone with Cosmos-owned checkpoint name mapping."""
+
+ # The enclosing Cosmos model maps and packs checkpoint weights. Disabling
+ # Nemotron-H's checkpoint-specific hook prevents those names from being
+ # remapped a second time when AutoWeightsLoader descends into this module.
+ load_weights = None # type: ignore[assignment]
+
+
+class Cosmos3EdgeForCausalLM(nn.Module):
+ """Minimal CausalLM wrapper for the Cosmos3 Edge dense text model."""
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__()
+ hf_config = vllm_config.model_config.hf_config
+ text_config = getattr(hf_config, "text_config", hf_config)
+ vllm_config = vllm_config.with_hf_config(text_config)
+
+ self.vllm_config = vllm_config
+ self.model_config = vllm_config.model_config
+ self.quant_config = vllm_config.quant_config
+ self.config = text_config
+ pattern = text_config.hybrid_override_pattern
+ unsupported = set(pattern) - {"*", "-"}
+ if unsupported:
+ raise ValueError(
+ "Cosmos3 Edge only supports attention (`*`) and MLP (`-`) "
+ f"layers, but found: {sorted(unsupported)}"
+ )
+ if len(pattern) != text_config.num_hidden_layers:
+ raise ValueError(
+ "hybrid_override_pattern must contain one entry per layer: "
+ f"expected {text_config.num_hidden_layers}, "
+ f"got {len(pattern)}"
+ )
+
+ self.model = Cosmos3EdgeTextModel(
+ vllm_config=vllm_config,
+ prefix=maybe_prefix(prefix, "model"),
+ decoder_layer_types={
+ "*": Cosmos3EdgeAttentionDecoderLayer,
+ "-": NemotronHMLPDecoderLayer,
+ },
+ )
+ self.lm_head = ParallelLMHead(
+ text_config.vocab_size,
+ text_config.hidden_size,
+ quant_config=self.quant_config,
+ prefix=maybe_prefix(prefix, "lm_head"),
+ )
+ self.logits_processor = LogitsProcessor(text_config.vocab_size)
+ self.make_empty_intermediate_tensors = (
+ self.model.make_empty_intermediate_tensors
+ )
+
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+ return self.model.embed_input_ids(input_ids)
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ **kwargs,
+ ):
+ return self.model(
+ input_ids=input_ids,
+ positions=positions,
+ intermediate_tensors=intermediate_tensors,
+ inputs_embeds=inputs_embeds,
+ )
+
+ def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
+ return self.logits_processor(self.lm_head, hidden_states)
+
+
+def _cosmos3_edge_diffusers_prefix_map() -> dict[str, str]:
+ """Map Diffusers blocks to interleaved Nemotron-H layers.
+
+ The checkpoint stores attention and MLP in one ``layers.N`` block, while
+ the Nemotron-H model represents them as separate decoder layers. Therefore,
+ checkpoint block ``N`` maps its attention to layer ``2N`` and its MLP to
+ layer ``2N + 1``.
+ """
+ mappings = {
+ "embed_tokens.": "language_model.model.embed_tokens.",
+ "norm.": "language_model.model.norm_f.",
+ }
+ for physical_idx in range(28):
+ attention_idx = 2 * physical_idx
+ mlp_idx = attention_idx + 1
+ source_prefix = f"layers.{physical_idx}"
+ attention_prefix = f"language_model.model.layers.{attention_idx}"
+ mlp_prefix = f"language_model.model.layers.{mlp_idx}"
+ mappings.update(
+ {
+ f"{source_prefix}.input_layernorm.": f"{attention_prefix}.norm.",
+ f"{source_prefix}.self_attn.qkv_proj.": (
+ f"{attention_prefix}.mixer.qkv_proj."
+ ),
+ f"{source_prefix}.self_attn.o_proj.": (
+ f"{attention_prefix}.mixer.o_proj."
+ ),
+ f"{source_prefix}.post_attention_layernorm.": (f"{mlp_prefix}.norm."),
+ f"{source_prefix}.mlp.": f"{mlp_prefix}.mixer.",
+ }
+ )
+ return mappings
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+ Cosmos3EdgeMultiModalProcessor,
+ info=Cosmos3EdgeProcessingInfo,
+ dummy_inputs=Cosmos3EdgeDummyInputsBuilder,
+)
+class Cosmos3EdgeForConditionalGeneration(
+ nn.Module,
+ SupportsMultiModal,
+ SupportsPP,
+ SupportsMRoPE,
+):
+ """
+ Cosmos3 Edge model with a SigLIP2 vision encoder.
+
+ Architecture:
+ - self.visual: SigLIP2 encoder + patch merger + projector
+ - self.language_model: Cosmos3EdgeForCausalLM (pure attention + RoPE)
+ """
+
+ hf_to_vllm_mapper = WeightsMapper(
+ orig_to_new_stacked={
+ ".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
+ ".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
+ ".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
+ },
+ orig_to_new_prefix={
+ **_cosmos3_edge_diffusers_prefix_map(),
+ "proj_in.": None,
+ "proj_out.": None,
+ "time_embedder.": None,
+ "action_proj_in.": None,
+ "action_proj_out.": None,
+ "audio_modality_embed": None,
+ "action_modality_embed": None,
+ "model.visual.": "visual.encoder.",
+ "model.projector.": "visual.projector.",
+ "lm_head.": "language_model.lm_head.",
+ "model.language_model.": "language_model.model.",
+ },
+ orig_to_new_substr={
+ "_moe_gen": None,
+ ".add_q_proj.": None,
+ ".add_k_proj.": None,
+ ".add_v_proj.": None,
+ ".to_add_out.": None,
+ ".norm_added_q.": None,
+ ".norm_added_k.": None,
+ ".self_attn.to_q.": ".self_attn.q_proj.",
+ ".self_attn.to_k.": ".self_attn.k_proj.",
+ ".self_attn.to_v.": ".self_attn.v_proj.",
+ ".self_attn.to_out.": ".self_attn.o_proj.",
+ "language_model.embeddings": "language_model.embed_tokens",
+ },
+ )
+
+ packed_modules_mapping = {
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+ }
+
+ # Cosmos3 Edge unified Diffusers checkpoints store reasoner weights across
+ # transformer/ and vision_encoder/. Match both while excluding VAE weights.
+ allow_patterns_overrides = ["[tv]*er/*.safetensors"]
+
+ supports_encoder_tp_data = True
+
+ @classmethod
+ def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+ if modality.startswith("image"):
+ return "<|vision_start|><|image_pad|><|vision_end|>"
+ if modality.startswith("video"):
+ return "<|vision_start|><|video_pad|><|vision_end|>"
+ raise ValueError(f"Unsupported modality: {modality}")
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
+ super().__init__()
+
+ config = vllm_config.model_config.hf_config
+ quant_config = vllm_config.quant_config
+ multimodal_config = vllm_config.model_config.multimodal_config
+
+ self.config = config
+ self.multimodal_config = multimodal_config
+ self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
+
+ with self._mark_tower_model(vllm_config, {"image", "video"}):
+ self.visual = Cosmos3EdgeVisionModel(
+ vision_config=config.vision_config,
+ projector_config=config.projector_config,
+ quant_config=quant_config,
+ prefix=maybe_prefix(prefix, "visual"),
+ )
+
+ with self._mark_language_model(vllm_config):
+ self.language_model = Cosmos3EdgeForCausalLM(
+ vllm_config=vllm_config,
+ prefix=maybe_prefix(prefix, "language_model"),
+ )
+
+ self.make_empty_intermediate_tensors = (
+ self.language_model.make_empty_intermediate_tensors
+ )
+
+ def _get_image_features(
+ self,
+ pixel_values: torch.Tensor,
+ grid_thw: torch.Tensor,
+ ) -> tuple[torch.Tensor, ...]:
+ """Run the complete vision tower and split by media item."""
+ image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
+ sizes = (grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()
+ return image_embeds.split(sizes)
+
+ def _parse_and_validate_image_input(
+ self, **kwargs: object
+ ) -> Qwen2_5_VLImageInputs | None:
+ pixel_values = kwargs.pop("pixel_values", None)
+ image_embeds = kwargs.pop("image_embeds", None)
+ image_grid_thw = kwargs.pop("image_grid_thw", None)
+
+ if pixel_values is None and image_embeds is None:
+ return None
+
+ if pixel_values is not None:
+ return Qwen2_5_VLImagePixelInputs(
+ type="pixel_values",
+ pixel_values=pixel_values,
+ image_grid_thw=image_grid_thw,
+ )
+
+ if image_embeds is not None:
+ return Qwen2_5_VLImageEmbeddingInputs(
+ type="image_embeds",
+ image_embeds=image_embeds,
+ image_grid_thw=image_grid_thw,
+ )
+
+ def _parse_and_validate_video_input(
+ self, **kwargs: object
+ ) -> Qwen2_5_VLVideoInputs | None:
+ pixel_values_videos = kwargs.pop("pixel_values_videos", None)
+ video_embeds = kwargs.pop("video_embeds", None)
+ video_grid_thw = kwargs.pop("video_grid_thw", None)
+ second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
+
+ if pixel_values_videos is None and video_embeds is None:
+ return None
+
+ if pixel_values_videos is not None:
+ return Qwen2_5_VLVideoPixelInputs(
+ type="pixel_values_videos",
+ pixel_values_videos=pixel_values_videos,
+ video_grid_thw=video_grid_thw,
+ second_per_grid_ts=second_per_grid_ts,
+ )
+
+ if video_embeds is not None:
+ return Qwen2_5_VLVideoEmbeddingInputs(
+ type="video_embeds",
+ video_embeds=video_embeds,
+ video_grid_thw=video_grid_thw,
+ )
+
+ def _process_image_input(
+ self, image_input: Qwen2_5_VLImageInputs
+ ) -> tuple[torch.Tensor, ...]:
+ grid_thw = image_input["image_grid_thw"]
+ assert grid_thw.ndim == 2
+
+ if image_input["type"] == "image_embeds":
+ image_embeds = image_input["image_embeds"].type(self.visual.dtype)
+ merge_size = self.visual.spatial_merge_size
+ sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
+ return image_embeds.split(sizes)
+
+ pixel_values = image_input["pixel_values"].type(self.visual.dtype)
+ if self.use_data_parallel:
+ return run_dp_sharded_mrope_vision_model(
+ self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
+ )
+ return self._get_image_features(pixel_values, grid_thw)
+
+ def _process_video_input(
+ self, video_input: Qwen2_5_VLVideoInputs
+ ) -> tuple[torch.Tensor, ...]:
+ grid_thw = video_input["video_grid_thw"]
+ assert grid_thw.ndim == 2
+
+ if video_input["type"] == "video_embeds":
+ video_embeds = video_input["video_embeds"].type(self.visual.dtype)
+ merge_size = self.visual.spatial_merge_size
+ sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
+ return video_embeds.split(sizes)
+
+ pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype)
+ if self.use_data_parallel:
+ return run_dp_sharded_mrope_vision_model(
+ self.visual, pixel_values_videos, grid_thw.tolist(), rope_type="rope_3d"
+ )
+ return self._get_image_features(pixel_values_videos, grid_thw)
+
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
+ modalities = {}
+ image_input = self._parse_and_validate_image_input(**kwargs)
+ if image_input is not None:
+ modalities["image"] = image_input
+ video_input = self._parse_and_validate_video_input(**kwargs)
+ if video_input is not None:
+ modalities["video"] = video_input
+ return modalities
+
+ def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
+ mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
+ if not mm_input_by_modality:
+ return None
+
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
+
+ for modality in mm_input_by_modality:
+ multimodal_input = mm_input_by_modality[modality]
+ if modality == "image":
+ image_embeddings = self._process_image_input(multimodal_input)
+ multimodal_embeddings += tuple(image_embeddings)
+ if modality == "video":
+ video_embeddings = self._process_video_input(multimodal_input)
+ multimodal_embeddings += tuple(video_embeddings)
+
+ return multimodal_embeddings
+
+ def forward(
+ self,
+ input_ids: torch.Tensor | None,
+ positions: torch.Tensor,
+ intermediate_tensors: IntermediateTensors | None = None,
+ inputs_embeds: torch.Tensor | None = None,
+ **kwargs: object,
+ ) -> torch.Tensor | IntermediateTensors:
+ if intermediate_tensors is not None:
+ inputs_embeds = None
+
+ return self.language_model.model(
+ input_ids=input_ids,
+ positions=positions,
+ intermediate_tensors=intermediate_tensors,
+ inputs_embeds=inputs_embeds,
+ )
+
+ def compute_logits(
+ self,
+ hidden_states: torch.Tensor,
+ ) -> torch.Tensor | None:
+ return self.language_model.compute_logits(hidden_states)
+
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+ loader = AutoWeightsLoader(self)
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+
+ def get_mm_mapping(self) -> MultiModelKeys:
+ return MultiModelKeys.from_string_field(
+ language_model="language_model",
+ connector="visual.projector",
+ tower_model="visual.",
+ )
+
+ def get_mrope_input_positions(
+ self,
+ input_tokens: list[int],
+ mm_features: list[MultiModalFeatureSpec],
+ ) -> tuple[torch.Tensor, int]:
+ return Qwen3VLForConditionalGeneration._get_mrope_input_positions(
+ input_tokens=input_tokens,
+ mm_features=mm_features,
+ config=self.config,
+ )
diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py
index bd5cd358d8..016bff18ec 100644
--- a/vllm/model_executor/models/nemotron_h.py
+++ b/vllm/model_executor/models/nemotron_h.py
@@ -19,7 +19,7 @@
"""Inference-only NemotronH model."""
import typing
-from collections.abc import Callable, Iterable
+from collections.abc import Callable, Iterable, Mapping
from itertools import islice
import torch
@@ -540,9 +540,22 @@ ALL_DECODER_LAYER_TYPES = {
}
-@support_torch_compile
+@support_torch_compile(
+ dynamic_arg_dims={
+ "input_ids": 0,
+ "positions": -1,
+ "intermediate_tensors": 0,
+ "inputs_embeds": 0,
+ }
+)
class NemotronHModel(nn.Module, EagleModelMixin):
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ def __init__(
+ self,
+ *,
+ vllm_config: VllmConfig,
+ prefix: str = "",
+ decoder_layer_types: Mapping[str, type[nn.Module]] | None = None,
+ ):
super().__init__()
config: NemotronHConfig = vllm_config.model_config.hf_config
@@ -562,11 +575,16 @@ class NemotronHModel(nn.Module, EagleModelMixin):
self.has_moe = "E" in config.hybrid_override_pattern
+ layer_types = decoder_layer_types or ALL_DECODER_LAYER_TYPES
+
def get_layer(prefix: str):
layer_idx = int(prefix.rsplit(".", 1)[1])
- layer_class = ALL_DECODER_LAYER_TYPES[
- config.hybrid_override_pattern[layer_idx]
- ]
+ layer_type = config.hybrid_override_pattern[layer_idx]
+ if layer_type not in layer_types:
+ raise ValueError(
+ f"Unsupported layer type {layer_type!r} at layer {layer_idx}"
+ )
+ layer_class = layer_types[layer_type]
return layer_class(
config=config,
layer_idx=layer_idx,
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 807c0dc40b..c8486dac5f 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -349,6 +349,10 @@ _MULTIMODAL_MODELS = {
"Cohere2VisionForConditionalGeneration",
),
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
+ "Cosmos3EdgeForConditionalGeneration": (
+ "cosmos3_edge",
+ "Cosmos3EdgeForConditionalGeneration",
+ ),
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index a848bbe142..c0f3396b0b 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -81,6 +81,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
cosmos3_omni="Cosmos3Config",
+ cosmos3_edge="Cosmos3EdgeConfig",
diffusion_gemma="DiffusionGemmaConfig",
deepseek_vl_v2="DeepseekVLV2Config",
deepseek_v32="DeepseekV3Config",
diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py
index f48b0dd6df..b64310507b 100644
--- a/vllm/transformers_utils/configs/__init__.py
+++ b/vllm/transformers_utils/configs/__init__.py
@@ -27,6 +27,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
+ "Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
+ "Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
+ "Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
+ "Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
@@ -107,6 +111,10 @@ __all__ = [
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
"Cosmos3Config",
+ "Cosmos3EdgeConfig",
+ "Cosmos3EdgeProjectorConfig",
+ "Cosmos3EdgeTextConfig",
+ "Cosmos3EdgeVisionConfig",
"DiffusionGemmaConfig",
"DiffusionGemmaTextConfig",
"DeepseekVLV2Config",
diff --git a/vllm/transformers_utils/configs/cosmos3_edge.py b/vllm/transformers_utils/configs/cosmos3_edge.py
new file mode 100644
index 0000000000..743e7d5deb
--- /dev/null
+++ b/vllm/transformers_utils/configs/cosmos3_edge.py
@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
+
+from .nemotron_h import NemotronHConfig
+
+
+def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
+ """Normalize the checkpoint's mRoPE configuration for vLLM."""
+ rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
+ mrope_section = rope_parameters.get("mrope_section")
+ if mrope_section is None:
+ return
+
+ rope_parameters["mrope_section"] = list(mrope_section)
+ rope_parameters.setdefault("mrope_interleaved", True)
+ config.rope_parameters = rope_parameters
+
+
+class Cosmos3EdgeTextConfig(NemotronHConfig):
+ """Dense Cosmos3 Edge text config backed by Nemotron-H layers."""
+
+ model_type = "cosmos3_edge_text"
+ has_no_defaults_at_init = True
+ ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
+
+ def __init__(
+ self,
+ num_hidden_layers: int = 28,
+ hidden_act: str = "relu2",
+ rms_norm_eps: float = 1e-5,
+ **kwargs,
+ ) -> None:
+ super().__init__(
+ num_hidden_layers=2 * num_hidden_layers,
+ hybrid_override_pattern="*-" * num_hidden_layers,
+ mlp_hidden_act=hidden_act,
+ layer_norm_epsilon=rms_norm_eps,
+ **kwargs,
+ )
+
+ def to_dict(self) -> dict:
+ config_dict = super().to_dict()
+ config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
+ config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
+ config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
+ config_dict.pop("hybrid_override_pattern", None)
+ return config_dict
+
+
+class Cosmos3EdgeProjectorConfig(PretrainedConfig):
+ model_type = "cosmos3_edge_projector"
+
+ def __init__(
+ self,
+ input_hidden_size: int = 1024,
+ use_postshuffle_norm: bool = False,
+ spatial_merge_size: int = 2,
+ merger_intermediate_size: int = 11520,
+ out_hidden_size: int = 1024,
+ **kwargs,
+ ) -> None:
+ super().__init__(**kwargs)
+ self.input_hidden_size = input_hidden_size
+ self.use_postshuffle_norm = use_postshuffle_norm
+ self.spatial_merge_size = spatial_merge_size
+ self.merger_intermediate_size = merger_intermediate_size
+ self.out_hidden_size = out_hidden_size
+
+
+class Cosmos3EdgeVisionConfig(Siglip2VisionConfig):
+ model_type = "cosmos3_edge_vision"
+
+
+class Cosmos3EdgeConfig(PretrainedConfig):
+ model_type = "cosmos3_edge"
+ keys_to_ignore_at_inference = ["past_key_values"]
+ has_no_defaults_at_init = True
+ sub_configs = {
+ "vision_config": Cosmos3EdgeVisionConfig,
+ "projector_config": Cosmos3EdgeProjectorConfig,
+ "text_config": Cosmos3EdgeTextConfig,
+ }
+
+ def __init__(
+ self,
+ text_config: Cosmos3EdgeTextConfig | dict | None = None,
+ vision_config: Cosmos3EdgeVisionConfig | dict | None = None,
+ projector_config: Cosmos3EdgeProjectorConfig | dict | None = None,
+ image_token_id: int = 19,
+ video_token_id: int = 18,
+ vision_start_token_id: int = 20,
+ vision_end_token_id: int = 21,
+ tie_word_embeddings: bool = False,
+ **kwargs,
+ ) -> None:
+ if text_config is None:
+ text_config = {}
+ if vision_config is None:
+ vision_config = {}
+ if projector_config is None:
+ projector_config = {}
+
+ self.text_config = (
+ Cosmos3EdgeTextConfig(**text_config)
+ if isinstance(text_config, dict)
+ else text_config
+ )
+ _normalize_mrope_parameters(self.text_config)
+
+ self.vision_config = (
+ Cosmos3EdgeVisionConfig(**vision_config)
+ if isinstance(vision_config, dict)
+ else vision_config
+ )
+ self.projector_config = (
+ Cosmos3EdgeProjectorConfig(**projector_config)
+ if isinstance(projector_config, dict)
+ else projector_config
+ )
+
+ # Qwen3-VL processing and M-RoPE read these attributes from the
+ # vision config, while this checkpoint stores them in projector config.
+ self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size
+ self.vision_config.temporal_patch_size = 1
+ self.vision_config.out_hidden_size = self.projector_config.out_hidden_size
+
+ self.image_token_id = image_token_id
+ self.video_token_id = video_token_id
+ self.vision_start_token_id = vision_start_token_id
+ self.vision_end_token_id = vision_end_token_id
+
+ super().__init__(
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+
+__all__ = [
+ "Cosmos3EdgeConfig",
+ "Cosmos3EdgeProjectorConfig",
+ "Cosmos3EdgeTextConfig",
+ "Cosmos3EdgeVisionConfig",
+]
diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py
index 60eb2922fd..e35218c4be 100644
--- a/vllm/transformers_utils/processors/__init__.py
+++ b/vllm/transformers_utils/processors/__init__.py
@@ -14,6 +14,7 @@ __all__ = [
"BagelProcessor",
"CheersProcessor",
"CohereASRProcessor",
+ "Cosmos3EdgeProcessor",
"DeepseekVLV2Processor",
"FireRedASR2Processor",
"FireRedLIDProcessor",
@@ -49,6 +50,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
"CheersProcessor": "vllm.transformers_utils.processors.cheers",
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
+ "Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
diff --git a/vllm/transformers_utils/processors/cosmos3_edge.py b/vllm/transformers_utils/processors/cosmos3_edge.py
new file mode 100644
index 0000000000..2644c6b17e
--- /dev/null
+++ b/vllm/transformers_utils/processors/cosmos3_edge.py
@@ -0,0 +1,356 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+import math
+
+import torch
+from torchvision.transforms import InterpolationMode
+from transformers import AutoTokenizer
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict
+from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor
+from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
+ Qwen3VLVideoProcessor,
+ Qwen3VLVideoProcessorInitKwargs,
+ get_image_size,
+ smart_resize,
+)
+from transformers.models.siglip2.image_processing_siglip2 import (
+ Siglip2ImageProcessor,
+ convert_image_to_patches,
+)
+from transformers.processing_utils import Unpack
+from transformers.utils import TensorType
+from transformers.video_utils import group_videos_by_shape, reorder_videos
+
+
+def round_by_factor(number: int, factor: int) -> int:
+ """Returns the closest integer to 'number' that is divisible by 'factor'."""
+ return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+ """Return the smallest multiple of ``factor`` greater than ``number``."""
+ return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+ """Return the largest multiple of ``factor`` less than ``number``."""
+ return math.floor(number / factor) * factor
+
+
+class Cosmos3EdgeImagesKwargs( # type: ignore[call-arg]
+ Siglip2ImageProcessor.valid_kwargs,
+ total=False,
+):
+ # global setting for all images, can be overridden by per-image kwargs
+ max_pixels: int | None
+ min_pixels: int | None
+
+ per_image_kwargs: list[dict] | None
+
+
+class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor):
+ resample = PILImageResampling.BICUBIC
+ valid_kwargs = Cosmos3EdgeImagesKwargs
+
+ def __init__(self, **kwargs: Unpack[Cosmos3EdgeImagesKwargs]):
+ super().__init__(**kwargs)
+
+ def _resize_image(
+ self,
+ image: torch.Tensor,
+ max_ratio=200,
+ resample: PILImageResampling | InterpolationMode | int | None = None,
+ **kwargs,
+ ) -> torch.Tensor:
+ """
+ Rescales the image so that the following conditions are met:
+
+ 1. Both dimensions (height and width) are divisible by 'factor'.
+ 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+ 3. The aspect ratio of the image is maintained as closely as possible.
+ """
+ image_min_pixels = kwargs.get("min_pixels") or self.size.get(
+ "shortest_edge", None
+ )
+ image_max_pixels = kwargs.get("max_pixels") or self.size.get(
+ "longest_edge", None
+ )
+ assert image_min_pixels is not None and image_max_pixels is not None, (
+ "When do_resize is True, min_pixels and max_pixels must be provided."
+ )
+ assert image_max_pixels >= image_min_pixels, (
+ "The max_pixels of image must be greater than or equal to min_pixels."
+ )
+
+ _, height, width = image.shape
+ aspect_ratio = max(height, width) / min(height, width)
+ if aspect_ratio > max_ratio:
+ raise ValueError(
+ f"absolute aspect ratio must be smaller than {max_ratio}, "
+ f"got {aspect_ratio}"
+ )
+ factor = self.merge_size * self.patch_size
+ h_bar = max(factor, round_by_factor(height, factor))
+ w_bar = max(factor, round_by_factor(width, factor))
+ if h_bar * w_bar > image_max_pixels:
+ beta = math.sqrt((height * width) / image_max_pixels)
+ h_bar = floor_by_factor(height / beta, factor)
+ w_bar = floor_by_factor(width / beta, factor)
+ elif h_bar * w_bar < image_min_pixels:
+ beta = math.sqrt(image_min_pixels / (height * width))
+ h_bar = ceil_by_factor(height * beta, factor)
+ w_bar = ceil_by_factor(width * beta, factor)
+
+ image = self.resize(
+ image,
+ size=SizeDict(height=h_bar, width=w_bar),
+ resample=resample,
+ )
+ return image
+
+ def _preprocess(
+ self,
+ images: list["torch.Tensor"],
+ do_resize: bool,
+ patch_size: int,
+ max_num_patches: int,
+ resample: PILImageResampling | InterpolationMode | int | None,
+ do_rescale: bool,
+ rescale_factor: float,
+ do_normalize: bool,
+ image_mean: float | list[float] | None,
+ image_std: float | list[float] | None,
+ return_tensors: str | TensorType | None,
+ **kwargs,
+ ) -> BatchFeature:
+ per_image_kwargs = kwargs.pop("per_image_kwargs", None)
+ pixel_values = []
+ spatial_shapes = []
+ for idx, image in enumerate(images):
+ # merge global kwargs with per-image overrides (if provided)
+ image_kwargs = kwargs.copy()
+ if (
+ per_image_kwargs is not None
+ and idx < len(per_image_kwargs)
+ and per_image_kwargs[idx]
+ ):
+ image_kwargs.update(per_image_kwargs[idx])
+ if do_resize:
+ image = self._resize_image(
+ image,
+ max_ratio=200,
+ resample=resample,
+ **image_kwargs,
+ )
+
+ image = self.rescale_and_normalize(
+ image, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+ )
+
+ # (channels, height, width) -> (patches, patch_size**2 * channels)
+ patches = convert_image_to_patches(image, patch_size)
+
+ num_patches_height = image.shape[1] // patch_size
+ num_patches_width = image.shape[2] // patch_size
+
+ spatial_shapes.append((num_patches_height, num_patches_width))
+ pixel_values.append(patches)
+
+ pixel_values = torch.cat(pixel_values, dim=0)
+
+ spatial_shapes = torch.tensor(spatial_shapes)
+ t_dim = torch.ones(
+ (spatial_shapes.shape[0], 1),
+ dtype=spatial_shapes.dtype,
+ device=spatial_shapes.device,
+ )
+ image_grid_thw = torch.cat([t_dim, spatial_shapes], dim=1)
+
+ batch_feature = BatchFeature(
+ data={
+ "pixel_values": pixel_values,
+ "image_grid_thw": image_grid_thw,
+ },
+ tensor_type=return_tensors,
+ )
+ return batch_feature
+
+
+class Cosmos3EdgeVideoProcessor(Qwen3VLVideoProcessor):
+ def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]):
+ super().__init__(**kwargs)
+
+ def _preprocess(
+ self,
+ videos: list[torch.Tensor],
+ do_convert_rgb: bool = True,
+ do_resize: bool = True,
+ size: SizeDict | None = None,
+ resample: PILImageResampling
+ | InterpolationMode
+ | int
+ | None = PILImageResampling.BICUBIC,
+ do_rescale: bool = True,
+ rescale_factor: float = 1 / 255.0,
+ do_normalize: bool = True,
+ image_mean: float | list[float] | None = None,
+ image_std: float | list[float] | None = None,
+ patch_size: int | None = None,
+ return_tensors: str | TensorType | None = None,
+ **kwargs,
+ ):
+ assert patch_size is not None
+
+ merge_size = self.merge_size
+ grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
+ resized_videos_grouped = {}
+
+ for shape, stacked_videos in grouped_videos.items():
+ B, T, C, H, W = stacked_videos.shape
+ num_frames, height, width = T, H, W
+ if do_resize:
+ assert size is not None
+ # 1. Both dimensions (height and width) are divisible by 'factor'.
+ # 2. Total pixels are within the configured min/max range.
+ resized_height, resized_width = smart_resize(
+ num_frames=num_frames,
+ height=height,
+ width=width,
+ temporal_factor=1,
+ factor=patch_size * merge_size,
+ min_pixels=size.shortest_edge,
+ max_pixels=size.longest_edge,
+ )
+ stacked_videos = stacked_videos.view(B * T, C, H, W)
+ stacked_videos = self.resize(
+ stacked_videos,
+ size=SizeDict(height=resized_height, width=resized_width),
+ resample=resample,
+ )
+ stacked_videos = stacked_videos.view(
+ B, T, C, resized_height, resized_width
+ )
+ resized_videos_grouped[shape] = stacked_videos
+ resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
+
+ # Group videos by size for further processing
+ # Resize may return videos with different sizes.
+ grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
+ processed_videos_grouped = {}
+ processed_grids = {}
+ for shape, stacked_videos in grouped_videos.items():
+ resized_height, resized_width = get_image_size(
+ stacked_videos[0], channel_dim=ChannelDimension.FIRST
+ )
+ # Fused rescale and normalize
+ stacked_videos = self.rescale_and_normalize(
+ stacked_videos,
+ do_rescale,
+ rescale_factor,
+ do_normalize,
+ image_mean,
+ image_std,
+ )
+ patches = stacked_videos
+
+ batch_size, grid_t, channel = patches.shape[:3]
+ grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+
+ # [1, 16, 3, 320, 608] -> [1, 16, 3, 20, 16, 38, 16]
+ patches = patches.view(
+ batch_size, # 0
+ grid_t, # 1
+ channel, # 2
+ grid_h, # 3
+ patch_size, # 4
+ grid_w, # 5
+ patch_size, # 6
+ )
+ # [1, 16, 3, 20, 16, 38, 16] -> [1, 16, 20, 38, 16, 16, 3]
+ # Result: [batch, grid_t, grid_h, grid_w, patch_h, patch_w, channel]
+ patches = patches.permute(0, 1, 3, 5, 4, 6, 2)
+
+ # [1, 16, 20, 38, 16, 16, 3] -> [1, 5120, 768]
+ flatten_patches = patches.reshape(
+ batch_size,
+ grid_t * grid_h * grid_w,
+ patch_size * patch_size * channel,
+ )
+
+ processed_videos_grouped[shape] = flatten_patches
+ processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
+
+ processed_videos = reorder_videos(
+ processed_videos_grouped, grouped_videos_index
+ )
+ processed_grids = reorder_videos(processed_grids, grouped_videos_index)
+ pixel_values_videos = torch.cat(processed_videos, dim=0)
+ video_grid_thw = torch.tensor(processed_grids)
+ data = {
+ "pixel_values_videos": pixel_values_videos,
+ "video_grid_thw": video_grid_thw,
+ }
+
+ return BatchFeature(data=data, tensor_type=return_tensors)
+
+
+class Cosmos3EdgeProcessor(Qwen3VLProcessor):
+ """Construct a processor for Cosmos3 Edge multimodal inputs.
+
+ This processor combines :class:`Cosmos3EdgeImageProcessor`,
+ :class:`Cosmos3EdgeVideoProcessor`, and a tokenizer into a single
+ interface. It inherits the text, image, and video input handling and the
+ decoding methods from :class:`Qwen3VLProcessor`, while using the Cosmos3
+ Edge-specific image and video preprocessing implementations.
+
+ Args:
+ image_processor: A :class:`Cosmos3EdgeImageProcessor` used to resize,
+ normalize, and patch image inputs.
+ tokenizer: The tokenizer used to encode and decode text.
+ video_processor: A :class:`Cosmos3EdgeVideoProcessor` used to resize,
+ normalize, and patch video inputs.
+ chat_template: An optional Jinja template used to render chat messages
+ into a tokenizable string.
+ """
+
+ attributes = ["image_processor", "tokenizer", "video_processor"]
+
+ # Transformers' auto loaders cannot resolve these vLLM-native image and
+ # video processors without remote code or global auto-class registration.
+ @classmethod
+ def _get_arguments_from_pretrained(
+ cls,
+ pretrained_model_name_or_path,
+ processor_dict=None,
+ **kwargs,
+ ):
+ component_kwargs = dict(kwargs)
+ component_kwargs.pop("trust_remote_code", None)
+
+ image_processor = Cosmos3EdgeImageProcessor.from_pretrained(
+ pretrained_model_name_or_path,
+ **component_kwargs,
+ )
+ tokenizer = AutoTokenizer.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=False,
+ **component_kwargs,
+ )
+ video_processor = Cosmos3EdgeVideoProcessor.from_pretrained(
+ pretrained_model_name_or_path,
+ **component_kwargs,
+ )
+
+ return [
+ image_processor,
+ tokenizer,
+ video_processor,
+ ]
+
+
+__all__ = [
+ "Cosmos3EdgeProcessor",
+]